{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../03_transformer_tutorial_1st_part/BERT_tutorial_and_attention_visualization')\n",
    "\n",
    "import torch\n",
    "from torch.optim import Adam\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "from dataset.wiki_dataset import BERTDataset\n",
    "from models.bert_model import BertForPreTraining, BertConfig, CrossEntropyLoss\n",
    "import tqdm\n",
    "import pandas as pd\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {}\n",
    "config[\"train_corpus_path\"] = \"./wiki_train.txt\"\n",
    "config[\"test_corpus_path\"] = \"./wiki_test.txt\"\n",
    "config[\"word2idx_path\"] = \"../03_transformer_tutorial_1st_part/BERT_tutorial_and_attention_visualization/corpus/bert_word2idx_extend.json\"\n",
    "config[\"output_path\"] = \"../03_transformer_tutorial_1st_part/BERT_tutorial_and_attention_visualization/bert_state_dict\"\n",
    "config[\"my_output_path\"] = \"./my_output_path\"\n",
    "config[\"batch_size\"] = 1\n",
    "config[\"max_seq_len\"] = 1000\n",
    "config[\"vocab_size\"] = 32162\n",
    "config[\"lr\"] = 2e-6\n",
    "config[\"num_workers\"] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_positional_encoding(hidden_dim, max_seq_len):\n",
    "    position_enc = np.array([\n",
    "        [pos / np.power(10000, 2 * i / hidden_dim) for i in range(hidden_dim)]\n",
    "        if pos != 0 else np.zeros(hidden_dim) for pos in range(max_seq_len)])\n",
    "\n",
    "    position_enc[1:, 0::2] = np.sin(position_enc[1:, 0::2])  # dim 2i\n",
    "    position_enc[1:, 1::2] = np.cos(position_enc[1:, 1::2])  # dim 2i+1\n",
    "    denominator = np.sqrt(np.sum(position_enc**2, axis=1, keepdims=True))\n",
    "    position_enc = position_enc / (denominator + 1e-8)\n",
    "    position_enc = torch.from_numpy(position_enc).type(torch.FloatTensor)\n",
    "    return position_enc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bertconfig: {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 384,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 1536,\n",
      "  \"max_position_embeddings\": 1024,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 6,\n",
      "  \"type_vocab_size\": 256,\n",
      "  \"vocab_size\": 32162\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading Dataset: 64it [00:00, 975.91it/s]\n",
      "Loading Dataset: 59it [00:00, 1047.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "optim_parameters: 109 Parameter containing:\n",
      "tensor([[-0.0107,  0.0084,  0.0057,  ..., -0.0071, -0.0067, -0.0083],\n",
      "        [-0.0414, -0.0023, -0.0234,  ...,  0.0392,  0.0179, -0.0034],\n",
      "        [-0.0220, -0.0275,  0.0082,  ...,  0.0178, -0.0168,  0.0062],\n",
      "        ...,\n",
      "        [-0.0389, -0.0327, -0.0053,  ...,  0.0146,  0.0213, -0.0433],\n",
      "        [ 0.0243, -0.0043, -0.0188,  ...,  0.0088,  0.0155,  0.0289],\n",
      "        [-0.0092,  0.0223,  0.0101,  ...,  0.0014,  0.0243, -0.0088]],\n",
      "       requires_grad=True) Parameter containing:\n",
      "tensor([[ 2.0700e-02,  2.9610e-02, -1.0974e-03,  1.0792e-02, -5.1642e-03,\n",
      "         -3.4115e-02, -6.7363e-03, -2.3167e-03,  2.0031e-03, -1.1687e-02,\n",
      "         -8.3131e-03, -2.1917e-02, -2.4903e-02, -3.1404e-02,  4.0429e-02,\n",
      "         -1.4183e-02, -1.1174e-03,  2.6756e-02, -4.0170e-02,  1.6238e-02,\n",
      "          1.9360e-02,  1.5132e-03, -6.1569e-03, -4.5144e-03, -1.1099e-02,\n",
      "         -2.8455e-03,  3.0869e-02, -2.2969e-02,  3.5826e-02, -1.1830e-02,\n",
      "          1.2079e-02,  3.1552e-02, -3.6165e-02, -2.0598e-02, -1.8479e-02,\n",
      "          3.0022e-03, -1.0403e-02, -7.9297e-03, -2.4221e-03, -1.0815e-02,\n",
      "          8.0832e-03, -4.7590e-03,  1.5959e-02, -2.7496e-02, -3.8913e-02,\n",
      "          5.7209e-03, -2.7327e-02, -1.7390e-02,  1.4945e-02, -3.7647e-02,\n",
      "         -2.8954e-02, -1.9210e-02, -2.1951e-02, -4.7384e-04, -1.9890e-02,\n",
      "         -2.2315e-02, -1.8532e-02, -1.0555e-02, -5.2964e-03, -2.1720e-02,\n",
      "         -4.7759e-03,  4.2151e-02, -8.0668e-03,  1.9483e-02,  2.2010e-02,\n",
      "          3.3929e-03, -4.8374e-03,  2.7399e-02, -1.3005e-02,  9.7302e-03,\n",
      "          9.5329e-03, -2.8056e-02,  1.7892e-02, -2.2200e-02,  1.0628e-02,\n",
      "         -1.8270e-02, -2.3396e-02, -9.5409e-04,  2.6247e-02, -1.9000e-03,\n",
      "         -4.6695e-02,  1.8021e-02, -3.3348e-02, -2.5081e-02,  9.1144e-03,\n",
      "         -3.4171e-03,  4.3836e-02,  6.4013e-03,  1.3208e-02,  2.9217e-02,\n",
      "          7.4605e-03, -1.9189e-02, -8.8199e-03,  3.6407e-02, -2.1831e-02,\n",
      "          1.3096e-02,  1.3507e-02,  2.7565e-02, -3.8239e-03, -3.1374e-02,\n",
      "         -6.3660e-03,  8.2896e-03, -1.0934e-02, -1.9491e-02,  1.4640e-02,\n",
      "         -9.6239e-03, -9.6118e-03, -2.8101e-02, -3.9595e-02,  2.7535e-02,\n",
      "         -1.4278e-02, -6.2694e-03, -1.3169e-02,  4.7686e-02,  2.8752e-02,\n",
      "         -2.0738e-02,  1.2088e-02, -4.0226e-02, -2.9216e-02,  3.7704e-03,\n",
      "         -2.5074e-02,  1.5618e-02,  1.2647e-02,  1.4763e-02,  5.8728e-03,\n",
      "         -1.7503e-02,  1.0473e-02, -1.8055e-02,  1.9556e-02,  3.7350e-02,\n",
      "          3.8226e-02, -2.9483e-02, -2.8629e-02,  2.0665e-02, -3.9826e-02,\n",
      "          1.1321e-02,  4.3246e-02,  5.0220e-04,  1.1608e-02, -4.1420e-03,\n",
      "         -7.2545e-03, -1.2373e-02,  8.4149e-03, -1.2650e-04, -5.9139e-02,\n",
      "         -2.0486e-02,  2.5189e-02,  5.1224e-02, -1.1444e-02,  3.4779e-02,\n",
      "          3.2505e-02, -7.3164e-03, -2.7209e-02, -2.4502e-02, -1.6292e-03,\n",
      "          2.3378e-02,  2.8106e-02, -1.1453e-02,  5.1333e-03, -6.4279e-03,\n",
      "          7.0452e-03,  3.1814e-03, -1.7186e-02, -1.5096e-02, -1.3812e-02,\n",
      "         -3.9227e-02,  3.0275e-02,  4.5128e-02,  2.3415e-02, -6.2499e-02,\n",
      "          2.8070e-02, -1.8229e-02,  4.4923e-02, -2.1776e-02, -4.0285e-02,\n",
      "         -3.3828e-03, -1.2337e-02, -1.8514e-02, -1.9540e-02,  1.3288e-02,\n",
      "          2.0764e-02, -4.7169e-03,  1.5820e-02,  7.6219e-03,  1.5495e-02,\n",
      "          1.6727e-02,  6.5558e-03,  4.4620e-02, -1.5210e-02, -3.1649e-02,\n",
      "         -1.0579e-02,  1.4842e-02, -2.3170e-03, -4.5106e-02,  4.9442e-03,\n",
      "         -2.0088e-02,  7.3615e-03,  3.3839e-02, -5.0807e-03, -2.6522e-03,\n",
      "          5.0261e-03, -2.0923e-03,  1.8822e-03, -3.3783e-02, -1.6235e-02,\n",
      "          4.2348e-02,  2.0073e-02, -1.1763e-02, -8.3680e-03, -7.4050e-03,\n",
      "         -3.0972e-03, -6.1785e-03, -1.8271e-02, -1.6534e-02, -2.0108e-02,\n",
      "          2.9172e-02, -1.5751e-02, -2.6569e-03, -1.5053e-02,  3.8418e-03,\n",
      "          2.7985e-03,  4.5149e-02, -1.2578e-02, -2.0602e-02, -5.3699e-03,\n",
      "          2.5163e-02,  6.5129e-03,  3.3423e-02, -3.5536e-03,  3.4946e-02,\n",
      "          6.4666e-03,  1.4367e-02, -1.8076e-02, -8.1034e-03,  3.5895e-02,\n",
      "         -1.5078e-02, -5.1220e-03,  1.4798e-02,  2.1678e-02,  6.9819e-03,\n",
      "          1.2256e-02, -2.0604e-02, -6.2484e-03, -8.7147e-03, -2.3741e-02,\n",
      "         -2.2128e-02, -1.7746e-02, -8.7348e-03,  3.1040e-02, -3.0042e-02,\n",
      "         -3.3299e-03,  2.2667e-02,  9.8581e-03, -3.0660e-02, -1.6782e-02,\n",
      "          2.2784e-02, -2.9698e-02, -3.2902e-02,  1.3172e-02, -5.8911e-02,\n",
      "          5.0804e-03, -6.4097e-03, -2.6803e-02, -4.8327e-03, -3.5506e-02,\n",
      "          1.8371e-02,  3.4001e-03, -9.6542e-03, -7.7105e-03,  5.4419e-02,\n",
      "         -4.3427e-04,  1.3250e-04, -2.7657e-03,  6.1944e-03,  3.7694e-02,\n",
      "         -4.4197e-02,  8.0834e-03,  2.6024e-02,  2.6636e-02,  5.0673e-02,\n",
      "         -2.1407e-02,  1.8719e-02,  2.4274e-03, -7.1204e-03,  1.2378e-02,\n",
      "          1.3928e-02,  1.8190e-02, -2.5808e-02, -6.8086e-03, -2.2097e-03,\n",
      "         -6.7428e-03, -2.4834e-02, -1.7707e-03,  1.0026e-02, -1.1538e-02,\n",
      "         -8.9186e-04,  9.3674e-03, -2.5773e-02, -4.9888e-03,  1.6646e-02,\n",
      "          2.2371e-02, -3.0858e-02, -1.3783e-02, -1.4218e-02, -1.8793e-02,\n",
      "          1.8754e-02, -2.6736e-02,  3.1724e-02,  9.2362e-03,  8.9957e-03,\n",
      "          1.5748e-02, -1.0534e-02, -4.1035e-02, -2.0140e-03, -6.4322e-03,\n",
      "         -3.8023e-02, -1.1892e-02,  1.7848e-02,  5.0809e-03, -5.6032e-03,\n",
      "          6.1368e-03, -2.8816e-02, -1.1104e-02, -2.3830e-03, -5.5673e-03,\n",
      "          3.1613e-02, -3.4986e-02, -2.0719e-02,  1.4932e-02, -1.9339e-02,\n",
      "         -6.4237e-03,  3.9108e-02,  1.3452e-02, -5.3786e-02,  6.5916e-03,\n",
      "         -4.4225e-02, -1.6491e-03,  3.1884e-02,  1.6707e-02, -7.9812e-03,\n",
      "          2.3276e-02, -6.4302e-03, -8.8077e-03,  1.7226e-02, -7.5881e-03,\n",
      "          1.5415e-02, -2.0109e-02, -2.2043e-03,  1.3202e-02, -5.5046e-03,\n",
      "          2.5170e-02, -1.9266e-02,  5.4205e-04,  9.1660e-03, -4.2059e-03,\n",
      "          2.2298e-02,  4.7071e-03, -4.2264e-02,  1.9212e-02, -6.8005e-04,\n",
      "         -2.2272e-02, -6.7013e-03, -6.5542e-03, -1.6114e-02,  1.8297e-02,\n",
      "         -2.0266e-02,  1.1982e-02, -1.6517e-02, -1.5821e-02,  1.0036e-02,\n",
      "          1.8501e-02,  1.7542e-02,  1.5561e-02,  4.4734e-02,  8.2161e-03,\n",
      "          1.6550e-02,  2.4104e-02, -1.5523e-02,  1.1447e-02,  3.1312e-02,\n",
      "         -7.4403e-03,  3.6471e-03,  1.3637e-02, -1.3195e-02],\n",
      "        [-1.5548e-02,  1.8859e-02, -2.9286e-05, -2.8870e-02,  2.8797e-02,\n",
      "          1.8077e-02, -2.2255e-02,  2.9055e-02,  2.2055e-02, -3.1019e-02,\n",
      "          2.8351e-02,  2.3638e-02, -9.5215e-03,  1.9569e-02, -3.6945e-02,\n",
      "         -7.0856e-04,  1.4154e-02, -9.7861e-03, -3.1893e-02, -6.3757e-03,\n",
      "          3.8950e-03,  8.6531e-03, -4.4112e-04, -3.8706e-02,  2.7145e-02,\n",
      "         -8.6322e-03, -1.1769e-02, -5.6748e-03, -6.7110e-03, -5.9416e-03,\n",
      "          1.8483e-02,  1.3200e-03,  1.3554e-02, -2.8037e-02, -5.4815e-03,\n",
      "          2.4588e-02, -2.0017e-02,  6.0367e-02,  7.2677e-04,  1.5967e-02,\n",
      "         -3.5357e-02, -1.6712e-02, -1.5857e-02, -3.4060e-03, -1.0265e-02,\n",
      "          4.0262e-03,  1.3805e-02,  3.0896e-02, -1.0128e-03,  1.5139e-02,\n",
      "          4.6603e-02,  4.4794e-02, -1.1438e-02, -2.9515e-03,  8.3089e-05,\n",
      "         -5.2413e-04, -3.2317e-03, -9.6671e-03,  9.1143e-03,  1.5411e-02,\n",
      "          9.6121e-03,  2.4070e-03,  3.8226e-02,  2.4371e-03,  4.3704e-03,\n",
      "         -4.3756e-03, -3.5800e-03,  1.3557e-02, -1.7287e-02,  2.6758e-02,\n",
      "         -1.7458e-02, -1.4113e-02,  1.2130e-02,  1.1244e-02, -2.5551e-02,\n",
      "          2.6436e-02, -5.5901e-03,  1.3839e-02, -4.6802e-02,  2.0968e-02,\n",
      "         -2.5396e-02, -1.3060e-02, -1.7465e-02,  1.0924e-02, -1.3833e-02,\n",
      "          3.3038e-03, -1.5462e-02,  1.5235e-02, -2.4622e-02,  3.6577e-02,\n",
      "         -6.0750e-03,  1.2148e-02, -5.5906e-03, -1.1539e-02,  3.0630e-03,\n",
      "         -5.6324e-02, -2.3652e-03, -5.1094e-03, -8.1037e-03, -1.4366e-02,\n",
      "         -1.6100e-02, -6.5410e-03,  3.9544e-02, -2.3947e-02,  8.6670e-03,\n",
      "         -1.6365e-02, -5.1787e-06,  1.3251e-02, -8.4248e-03, -1.6102e-02,\n",
      "         -8.5040e-03,  1.1396e-03, -2.1934e-02, -2.3109e-02, -2.0862e-02,\n",
      "         -3.2481e-03, -3.7802e-02, -1.2362e-02, -4.8057e-03, -1.3588e-03,\n",
      "          3.0344e-02, -3.5110e-02, -5.6127e-02, -5.9868e-03, -7.7000e-03,\n",
      "         -1.8773e-03,  1.4388e-03,  4.5566e-02,  8.1049e-03, -1.1872e-02,\n",
      "         -1.7931e-02,  5.5080e-03, -9.1518e-03, -1.0492e-02,  1.8293e-02,\n",
      "          3.0778e-02,  2.9980e-03,  8.8378e-03,  1.2041e-03, -1.6793e-02,\n",
      "         -2.9547e-02,  1.7595e-02, -6.6823e-03,  1.4945e-02,  2.1099e-02,\n",
      "         -1.3208e-02,  2.5762e-02,  9.5614e-03,  1.5897e-02, -1.0320e-02,\n",
      "          2.2988e-02, -1.5979e-02,  2.9151e-03,  1.6777e-03,  6.7158e-03,\n",
      "         -2.5414e-02, -1.7875e-02, -1.7964e-02,  2.1281e-02, -1.6062e-02,\n",
      "          1.1318e-04, -7.5602e-03,  5.0347e-03, -2.3317e-02,  2.5299e-02,\n",
      "         -2.5557e-03,  2.8242e-02,  2.1979e-02, -5.5233e-03,  5.0362e-03,\n",
      "         -1.6111e-02,  4.2091e-02,  2.0948e-02, -3.4624e-02,  3.3687e-02,\n",
      "          5.7204e-03, -6.4418e-03,  2.3625e-02,  2.7278e-02, -2.4077e-02,\n",
      "         -2.1502e-02, -5.0831e-04, -2.3712e-02, -1.7523e-02,  1.7938e-02,\n",
      "          5.5192e-03,  7.9712e-03, -4.1276e-02,  4.4441e-02,  2.9682e-02,\n",
      "          2.0396e-02,  4.6171e-02, -1.9835e-02,  2.8651e-02, -1.1284e-02,\n",
      "         -2.6378e-02, -8.7678e-03, -1.7455e-02,  1.3906e-02,  2.7356e-02,\n",
      "         -4.5881e-03,  2.9595e-03, -2.6694e-02, -2.6743e-02,  3.7567e-02,\n",
      "          1.1865e-02,  3.0878e-02, -2.8668e-03, -1.8607e-02,  6.9419e-03,\n",
      "         -1.9735e-02,  1.6575e-02, -5.4594e-03, -1.2375e-02, -1.8354e-02,\n",
      "         -1.5613e-02, -6.1050e-04, -9.3765e-03,  5.3728e-03,  2.0249e-02,\n",
      "          1.6467e-02,  1.3869e-02, -1.7350e-02, -3.0327e-02, -1.0773e-02,\n",
      "         -3.4165e-02,  7.7382e-03,  1.0869e-02,  7.8275e-03,  2.5740e-02,\n",
      "          3.6320e-02, -1.9110e-02, -2.9511e-02,  1.8258e-02,  4.3772e-03,\n",
      "          1.8967e-02,  4.6203e-03, -1.5045e-02,  6.2112e-02, -3.3674e-02,\n",
      "          5.2726e-04, -2.3299e-02,  5.5871e-03, -1.8930e-02,  2.5107e-03,\n",
      "         -5.7137e-03,  5.1418e-02,  2.8782e-02,  3.9237e-03, -1.9716e-02,\n",
      "          3.1814e-03, -1.3681e-02, -1.8417e-02,  1.5759e-02,  2.7880e-03,\n",
      "         -1.9466e-02,  1.1326e-02,  1.0578e-02,  2.7444e-03, -2.4752e-02,\n",
      "         -3.4731e-03, -3.9781e-03,  6.5127e-03, -1.0662e-02, -1.4969e-02,\n",
      "          6.9526e-03, -1.1653e-02,  1.3631e-02, -2.7268e-02, -4.2364e-03,\n",
      "         -8.6926e-03,  1.8454e-02, -1.0203e-02,  7.8121e-03,  2.6721e-03,\n",
      "          2.1879e-02,  3.5548e-02, -5.9879e-03, -1.3678e-02, -2.0491e-02,\n",
      "         -1.0631e-02,  1.2615e-03,  5.4420e-03, -3.3416e-02,  7.4161e-03,\n",
      "         -9.3883e-03, -1.2292e-02,  4.5970e-03,  2.4145e-02, -5.9830e-03,\n",
      "          8.5765e-04, -3.8210e-02,  1.9329e-03,  3.0519e-02,  4.8842e-04,\n",
      "         -8.7722e-03,  2.1579e-02, -1.4903e-02,  3.1195e-02,  3.0581e-02,\n",
      "          1.9401e-02, -4.1113e-02, -1.8453e-02,  3.1447e-02,  3.8027e-02,\n",
      "         -3.1588e-02,  1.6810e-02,  2.8408e-02, -2.3220e-02,  2.0414e-02,\n",
      "          2.1107e-02,  2.8763e-03, -4.0198e-02,  9.4295e-03,  1.5802e-02,\n",
      "         -2.8497e-02, -5.2816e-03, -1.1856e-02,  1.7924e-02,  3.3984e-02,\n",
      "          1.6511e-02,  3.3826e-02, -2.8769e-03, -9.9833e-03,  1.1726e-02,\n",
      "          9.7478e-03, -5.7044e-02,  2.9548e-03, -1.3928e-02, -1.4683e-02,\n",
      "         -1.6828e-02, -2.5050e-02,  9.2814e-03, -1.8956e-02,  1.5099e-02,\n",
      "         -1.8138e-02,  1.2910e-02,  7.0213e-03, -2.0025e-02, -1.5880e-02,\n",
      "          2.0948e-02,  2.4666e-03,  1.7060e-02, -2.6969e-02, -2.6980e-05,\n",
      "          1.5411e-03,  2.2280e-02, -9.6846e-03,  2.2010e-02,  9.6396e-03,\n",
      "          6.0778e-03,  1.7399e-02, -3.2637e-02,  1.0792e-02,  8.2669e-03,\n",
      "         -2.2473e-02, -4.0373e-02,  3.5493e-02, -5.2454e-03, -3.4235e-03,\n",
      "         -7.3684e-05,  4.7321e-03,  1.1349e-02, -1.9067e-02, -5.3725e-02,\n",
      "         -8.3925e-03, -1.7360e-02, -6.7310e-03, -4.3077e-04,  5.6446e-03,\n",
      "         -1.5034e-02,  3.0108e-03, -8.4245e-03, -1.5983e-02, -3.5092e-02,\n",
      "         -1.0845e-03,  2.0399e-02,  9.6338e-03, -1.8353e-03,  1.6078e-02,\n",
      "          6.0933e-03, -6.2363e-04, -8.7298e-04, -3.3165e-02]],\n",
      "       requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "bert_model = BertForPreTraining\n",
    "# 词量, 注意在这里实际字(词)汇量 = vocab_size - 20,\n",
    "# 因为前20个token用来做一些特殊功能, 如padding等等\n",
    "vocab_size = config[\"vocab_size\"]\n",
    "batch_size = config[\"batch_size\"]\n",
    "# 学习率\n",
    "lr = config[\"lr\"]\n",
    "# 是否使用GPU\n",
    "cuda_condition = torch.cuda.is_available()\n",
    "device = torch.device(\"cuda:0\" if cuda_condition else \"cpu\")\n",
    "# 限定的单句最大长度\n",
    "max_seq_len = config[\"max_seq_len\"]\n",
    "# 初始化超参数的配置\n",
    "bertconfig = BertConfig(vocab_size_or_config_json_file=config[\"vocab_size\"])\n",
    "print(\"bertconfig:\", bertconfig)\n",
    "# 初始化bert模型\n",
    "bert_model = bert_model(config=bertconfig)\n",
    "bert_model.to(device)\n",
    "# 初始化训练数据集\n",
    "train_dataset = BERTDataset(corpus_path=config[\"train_corpus_path\"],\n",
    "                            word2idx_path=config[\"word2idx_path\"],\n",
    "                            seq_len=max_seq_len,\n",
    "                            hidden_dim=bertconfig.hidden_size,\n",
    "                            on_memory=True,\n",
    "                            )\n",
    "# 初始化训练dataloader\n",
    "train_dataloader = DataLoader(train_dataset,\n",
    "                                    batch_size=batch_size,\n",
    "                                    num_workers=config[\"num_workers\"],\n",
    "                                    collate_fn=lambda x: x)\n",
    "# 初始化测试数据集\n",
    "test_dataset = BERTDataset(corpus_path=config[\"test_corpus_path\"],\n",
    "                            word2idx_path=config[\"word2idx_path\"],\n",
    "                            seq_len=max_seq_len,\n",
    "                            hidden_dim=bertconfig.hidden_size,\n",
    "                            on_memory=True,\n",
    "                            )\n",
    "# 初始化测试dataloader\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=batch_size,\n",
    "                                    num_workers=config[\"num_workers\"],\n",
    "                                    collate_fn=lambda x: x)\n",
    "# 初始化positional encoding\n",
    "self_positional_enc = init_positional_encoding(hidden_dim=bertconfig.hidden_size,\n",
    "                                                    max_seq_len=max_seq_len)\n",
    "# 拓展positional encoding的维度为[1, max_seq_len, hidden_size]\n",
    "self_positional_enc = torch.unsqueeze(self_positional_enc, dim=0)\n",
    "\n",
    "# 列举需要优化的参数并传入优化器\n",
    "optim_parameters = list(bert_model.parameters())\n",
    "print(\"optim_parameters:\", len(optim_parameters), optim_parameters[0], optim_parameters[-2])\n",
    "optimizer = torch.optim.Adam(optim_parameters, lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'text1': '数学',\n",
       "  'text2': '数学\\n\\n数学是利用符号语言研究数量、结构、变化以及空间等概念的一门学科，从某种角度看属于形式科学的一种。数学透过抽象化和逻辑推理的使用，由计数、计算、量度和对物体形状及运动的观察而产生。数学家们拓展这些概念，为了公式化新的猜想以及从选定的公理及定义中建立起严谨推导出的定理。\\n\\n基础数学的知识与运用总是个人与团体生活中不可或缺的一环。对数学基本概念的完善，早在古埃及、美索不达米亚及古印度内的古代数学文本便可观见，而在古希腊那里有更为严谨的处理。从那时开始，数学的发展便持续不断地小幅进展，至16世纪的文艺复兴时期，因为新的科学发现和数学革新两者的交互，致使数学的加速发展，直至今日。数学并成为许多国家及地区的教育范畴中的一部分。\\n\\n今日，数学使用在不同的领域中，包括科学、工程、医学、经济学和金融学等。数学对这些领域的应用通常被称为应用数学，有时亦会激起新的数学发现，并导致全新学科的发展，例如物理学的实质性发展中建立的某些理论激发数学家对于某些问题的不同角度的思考。数学家也研究纯数学，就是数学本身的实质性内容，而不以任何实际应用为目标。虽然许多研究以纯数学开始，但其过程中也发现许多应用之处。\\n\\n西方语言中“数学”（）一词源自于古希腊语的（），其有“学习”、“学问”、“科学”，以及另外还有个较狭义且技术性的意思－「数学研究」，即使在其语源内。其形容词（），意思为\"和学习有关的\"或\"用功的\"，亦会被用来指\"数学的\"。其在英语中表面上的复数形式，及在法语中的表面复数形式\\'，可溯至拉丁文的中性复数\\'，由西塞罗译自希腊文复数（），此一希腊语被亚里士多德拿来指「万物皆数」的概念。\\n\\n汉字表示的「数学」一词大约产生于中国宋元时期。多指象数之学，但有时也含有今天上的数学意义，例如，秦九韶的《数学九章》（《永乐大典》记，即《数书九章》也被宋代周密所著的《癸辛杂识》记为《数学大略》）、《数学通轨》（明代柯尚迁著）、《数学钥》（清代杜知耕著）、《数学拾遗》（清代丁取忠撰）。直到1939年，经过中国数学名词审查委员会研究“算学”与“数学”两词的使用状况后，确认以“数学”表示今天意义上的数学含义。\\n\\n数学有着久远的历史。它被认为起源于人类早期的生产活动：中国古代的六艺之一就有「数」，数学一词在西方有希腊语词源（mathematikós），意思是“学问的基础”，源于（máthema，“科学，知识，学问”）。\\n\\n史前的人类就已尝试用自然的法则来衡量物质的多少、时间的长短等抽象的数量关系，比如时间单位有日、季节和年等。算术（加减乘除）也自然而然地产生了。古代的石碑及泥版亦证实了当时已有几何的知识。\\n\\n更进一步则需要写作或其他可记录数字的系统，如符木或于印加帝国内用来储存数据的奇普。历史上曾有过许多不同的记数系统。\\n\\n在最初有历史记录的时候，数学内的主要原理是为了做税务和贸易等相关计算，为了解数字间的关系，为了测量土地，以及为了预测天文事件而形成的。这些需要可以简单地被概括为数学对数量、结构、空间及时间方面的研究。\\n\\n到了16世纪，算术、初等代数以及三角学等初等数学已大体完备。17世纪变量概念的产生使人们开始研究变化中的量与量的互相关系和图形间的互相变换，微积分的概念也在此时形成。随着数学转向形式化，为研究数学基础而产生的集合论和数理逻辑等也开始发展。数学的重心从求解实际问题转变到对一般形式上的思考。\\n\\n从古至今，数学便一直不断地延展，且与科学有丰富的相互作用，两者的发展都受惠于彼此。在历史上有著许多数学发现，并且直至今日都不断地有新的发现。据Mikhail B. Sevryuk于2006年1月的期刊中所说，「存放于数学评论资料库中论文和书籍的数量自1940年（数学评论的创刊年份）现已超过了一百九十万份，而且每年还增加超过七万五千份。此一学海的绝大部份为新的数学定理及其证明。」\\n\\n每当有涉及数量、结构、空间及变化等方面的困难问题时，通常就需要用到数学工具去解决问题，而这往往也拓展了数学的研究范畴。一开始，数学的运用可见于贸易、土地测量及之后的天文学。今日，所有的科学都存在著值得数学家研究的问题，且数学本身亦给出了许多的问题。牛顿和莱布尼兹是微积分的发明者，费曼发明了费曼路径积分，这是推理及物理洞察二者的产物，而今日的弦理论亦引申出新的数学。一些数学只和生成它的领域有关，且用来解答此领域的更多问题。但一般被一领域生成的数学在其他许多领域内也十分有用，且可以成为一般的数学概念。即使是「最纯的」数学通常亦有实际的用途，此一非比寻常的事实，被1963年诺贝尔物理奖得主维格纳称为「数学在自然科学中不可想像的有效性」。\\n\\n如同大多数的研究领域，科学知识的爆发导致了数学的专业化。主要的分歧为纯数学和应用数学。在应用数学内，又被分成两大领域，并且变成了它们自身的学科——统计学和电脑科学。\\n\\n许多数学家谈论数学的\"优美\"，其内在的美学及美。「简单」和「一般化」即为美的一种。另外亦包括巧妙的证明，如欧几里得对存在无限多质数的证明；又或者是加快计算的数值方法，如快速傅立叶变换。高德菲·哈罗德·哈代在《一个数学家的自白》一书中表明他相信单单是美学上的意义，就已经足够作为纯数学研究的正当理由。\\n\\n我们现今所使用的大部分数学符号在16世纪后才被发明出来的。在此之前，数学以文字的形式书写出来，这种形式会限制了数学的发展。现今的符号使得数学对于专家而言更容易掌握，但初学者却常对此望而却步。它被极度的压缩：少量的符号包含著大量的讯息。如同音乐符号一般，现今的数学符号有明确的语法，并且有效地对讯息作编码，这是其他书写方式难以做到的。符号化和形式化使得数学迅速发展，并帮助各个科学领域建立基础支撑理论。\\n\\n数学语言亦对初学者而言感到困难。如“或”和“只”这些字有著比日常用语更精确的意思。亦困恼著初学者的，如“开放”和“域”等字在数学里有著特别的意思。数学术语亦包括如“同胚”及“可积性”等专有名词。但使用这些特别符号和专有术语是有其原因的：数学需要比日常用语更多的精确性。数学家将此对语言及逻辑精确性的要求称为「严谨」。但在现实应用中，舍弃一些严谨性往往会得到更好的结果。\\n\\n严谨是数学证明中很重要且基本的一部份。数学家希望他们的定理以系统化的推理依著公理被推论下去。这是为了避免依著不可靠的直观而推出错误的「定理」，而这情形在历史上曾出现过许多的例子。在数学中被期许的严谨程度因著时间而不同：希腊人期许著仔细的论证，但在牛顿的时代，所使用的方法则较不严谨。牛顿为了解决问题所做的定义，到了十九世纪才重新以小心的分析及正式的证明来处理。今日，数学家们则持续地在争论电脑辅助证明的严谨度。当大量的计算难以被验证时，其证明亦很难说是足够地严谨。\\n\\n公理在传统的思想中是「不证自明的真理」，但这种想法是有问题的。在形式上，公理只是一串符号，其只对可以由公理系统导出的公式之内容有意义。希尔伯特计划即是想将所有的数学放在坚固的公理基础上，但依据哥德尔不完备定理，每一相容且能蕴涵皮亚诺公理的公理系统必含有一不可决定的公式；因而所有数学的最终公理化是不可能的。尽管如此，数学常常被想像成只是某种公理化的集合论，在此意义下，所有数学叙述或证明都可以写成集合论的公式。\\n\\n卡尔·弗里德里希·高斯称数学为「科学的皇后」。在拉丁原文\\'，以及其德语\\'中，对应于「科学」的单字的意思皆为知识（领域）。而实际上，science一词在英语内本来就是这个意思，且无疑问地数学在此意义下确实是一门「科学」。将科学限定在自然科学则是在此之后的事。若认为科学是只指物理的世界时，则数学，或至少是纯数学，不会是一门科学。爱因斯坦曾如此描述：「数学定律越和现实有关，它们越不确定；若它们越是确定的话，它们和现实越不会有关。」\\n\\n许多哲学家相信数学在经验上不具可否证性，且因此不是卡尔·波普尔所定义的科学。但在1930年代时，在数理逻辑上的重大进展显示数学不能归并至逻辑内，且波普尔推断「大部份的数学定律，如物理及生物学一样，是假设演绎的：纯数学因此变得更接近其假设为猜测的自然科学，比它现在看起来更接近。」然而，其他的思想家，如较著名的拉卡托斯，便提供了一个关于数学本身的可否证性版本。\\n\\n另一观点则为某些科学领域（如理论物理）是其公理为尝试著符合现实的数学。而事实上，理论物理学家齐曼（John Ziman）即认为科学是一种公众知识，因此亦包含著数学。在任何的情况下，数学和物理科学的许多领域都有著很多相同的地方，尤其是从假设所得的逻辑推论之探索。直觉和实验在数学和科学的猜想建构上皆扮演著重要的角色。实验数学在数学中的重要性正持续地在增加，且计算和模拟在科学及数学中所扮演的角色也越来越加重，减轻了数学不使用科学方法的缺点。在史蒂芬·沃尔夫勒姆2002年的著作《一种新科学》中他提出，计算数学应被视为其自身的一科学领域来探索。\\n\\n数学家对此的态度并不一致。一些研究应用数学的数学家觉得他们是科学家，而那些研究纯数学的数学家则时常觉得他们是在一门较接近逻辑的领域内工作，且因此基本上是个哲学家。许多数学家认为称他们的工作是一种科学，是低估了其美学方面的重要性，以及其做为七大博雅教育之一的历史；另外亦有人认为若忽略其与科学之间的关联，是假装没看到数学和其在科学与工程之间的交互影响，进而促进了数学在许多科学上的发展此一事实。这两种观点之间的差异在哲学上产生了数学是「被创造」（如艺术）或是「被发现」（如科学）的争议。大学院系划分中常见「科学和数学系」，这指出了这两个领域被看作有紧密联系而非一样。实际上，数学家通常会在大体上与科学家合作，但在细节上却会分开。此争议亦是数学哲学众多议题的其中一个。\\n\\n如上所述，数学主要的学科最先产生于商业上计算的需要、了解数字间的关系、测量土地及预测天文事件。这四种需要大致地与数量、结构、空间及变化（即算术、代数、几何及分析）等数学上广泛的子领域相关连著。除了上述主要的关注之外，亦有用来探索由数学核心至其他领域上之间的连结的子领域：至逻辑、至集合论（基础）、至不同科学的经验上的数学（应用数学）、及较近代的至不确定性的严格研究。\\n为了阐明数学基础，数学逻辑和集合论等领域被发展了出来。\\n\\n数学逻辑专注于将数学置在一坚固的公理架构上，并研究此一架构的结果。就数学逻辑本身而言，其为哥德尔第二不完备定理所属的领域，而这或许是逻辑中最广为流传的成果－总存在一不能被证明而又为真的定理。\\n\\n现代逻辑被分成递归论、模型论和证明论，且和理论电脑科学有著密切的关连性，千禧年大奖难题中的P/NP问题就是理论电脑科学中的著名问题。\\n\\n数量的研究起于数，一开始为熟悉的自然数及整数与被描述在算术内的自然数及整数的算术运算。整数更深的性质于数论中有详细的研究，此一理论包括了如费马最后定理等著名的结果。数论还包括两个被广为探讨的未解问题：孪生质数猜想及哥德巴赫猜想。\\n\\n当数系更进一步发展时，整数被视为有理数的子集，而有理数则包含于实数中，连续的量即是以实数来表示的。实数则可以被进一步广义化成复数。数的进一步广义化可以持续至包含四元数及八元数。从自然数亦可以推广到超限数，它形式化了计数至无限的这一概念。另一个研究的领域为大小，这个导致了基数和之后对无限的另外一种概念：阿列夫数，它允许无限集合之间的大小可以做有意义的比较。\\n\\n许多如数及函数的集合等数学物件都有著内含的结构。这些物件的结构性质被探讨于群、环、-{zh-cn:域;zh-tw:体}-等抽象系统中，该些物件事实上也就是这样的系统。此为代数的领域。在此有一个很重要的概念，即广义化至向量空间的向量，它于线性代数中被研究。向量的研究结合了数学的三个基本领域：数量、结构及空间。向量分析则将其扩展至第四个基本的领域内，即变化。\\n\\n创立于二十世纪三十年代的法国的布尔巴基学派认为：纯粹数学，是研究抽象结构的理论。\\n结构，就是以初始概念和公理出发的演绎系统。\\n布尔巴基学派认为，有三种基本的抽象结构：代数结构（群，环，域……），序结构（偏序，全序……），拓扑结构（邻域，极限，连通性，维数……）。\\n\\n空间的研究源自于几何－尤其是欧几里得几何。三角学则结合了空间及数，且包含有著名的勾股定理。现今对空间的研究更推广到了更高维的几何、非欧几里得几何（其在广义相对论中扮演著核心的角色）及拓扑学。数和空间在解析几何、微分几何和代数几何中都有著很重要的角色。在微分几何中有著纤维丛及流形上的微积分等概念。在代数几何中有著如多项式方程的解集等几何物件的描述，结合了数和空间的概念；亦有著拓扑群的研究，结合了结构与空间。李群被用来研究空间、结构及变化。在其许多分支中，拓扑学可能是二十世纪数学中有著最大进展的领域，并包含有存在已久的庞加莱猜想，以及有争议的四色定理。庞加莱猜想已在2006年确认由俄罗斯数学家格里戈里·佩雷尔曼证明，而四色定理已在1976年由凯尼斯·阿佩尔和沃夫冈·哈肯用电脑证明，而从来没有由人力来验证过。\\n\\n了解及描述变化在自然科学里是一普遍的议题，而微积分更为研究变化的有利工具。函数诞生于此，做为描述一变化的量的核心概念。对于实数及实变函数的严格研究为实分析，而复分析则为复数的等价领域。黎曼猜想－数学最基本的未决问题之一－便是以复分析来描述的。泛函分析注重在函数的（一般为无限维）空间上。泛函分析的众多应用之一为量子力学。许多的问题很自然地会导出一个量与其变化率之间的关系，而这在微分方程中被研究。在自然界中的许多现象可以被动力系统所描述；混沌理论则是对系统的既不可预测而又是决定的行为作明确的描述。\\n离散数学是指对理论电脑科学最有用处的数学领域之总称，这包含有可计算理论、计算复杂性理论及资讯理论。可计算理论检验电脑的不同理论模型之极限，这包含现知最有力的模型－图灵机。复杂性理论研究可以由电脑做为较易处理的程度；有些问题即使理论是可以以电脑解出来，但却因为会花费太多的时间或空间而使得其解答仍然不为实际上可行的，尽管电脑硬体的快速进步。最后，资讯理论专注在可以储存在特定媒介内的资料总量，且因此有压缩及熵等概念。\\n\\n作为一相对较新的领域，离散数学有许多基本的未解问题。其中最有名的为P/NP问题－千禧年大奖难题之一。一般相信此问题的解答是否定的。\\n\\n应用数学思考将抽象的数学工具运用在解答科学、工商业及其他领域上之现实问题。应用数学中的一重要领域为统计学，它利用机率论为其工具并允许对含有机会成分的现象进行描述、分析与预测。大部份的实验、调查及观察研究需要统计对其资料的分析。（许多的统计学家并不认为他们是数学家，而比较觉得是合作团体的一份子。）数值分析研究有什么计算方法，可以有效地解决那些人力所限而算不出的数学问题；它亦包含了对计算中舍入误差或其他来源的误差之研究。\\n\\n数学奖通常和其他科学的奖项分开。数学上最有名的奖为菲尔兹奖，创立于1936年，每四年颁奖一次。它通常被认为是数学的诺贝尔奖。另一个国际上主要的奖项为阿贝尔奖，创立于2003年。两者都颁奖于特定的工作主题，包括数学新领域的创新或已成熟领域中未解决问题的解答。著名的23个问题，称为希尔伯特的23个问题，于1900年由德国数学家大卫·希尔伯特所提出。这一连串的问题在数学家之间有著极高的名望，且至少有九个问题已经被解答了出来。另一新的七个重要问题，称为千禧年大奖难题，发表于2000年。对其每一个问题的解答都有著一百万美元的奖金，而当中只有一个问题（黎曼猜想）和希尔伯特的问题重复。\\n\\n\\n\\n'},\n",
       " {'text1': '哲学',\n",
       "  'text2': '哲学\\n\\n哲学（）是研究普遍的、根本的问题的学科，包括存在、知识、价值、理智、心灵、语言等领域。哲学与其他学科的不同是其批判的方式、通常是系统化的方法，并以理性论证为基础。在日常用语中，其也可被引申为个人或团体的最基本信仰、概念或态度。\\n\\n英语词语（）源于古希腊语中的，意思为「爱智慧」，有时也译为「智慧的朋友」，该词由（philos，爱）的派生词（Philein，去爱）和（Sophia，智慧）组合而成。一般认为，古希腊思想家毕达哥拉斯最先在著作中引入“哲学家”和“哲学”这两个术语。\\n\\n“哲”一词在中国起源很早，如“孔门十哲”，“古圣先哲”等词，“哲”或“哲人”，专指那些善于思辨，学问精深者，即西方近世“哲学家”，“思想家”之谓。在《易经》当中已经开始讨论哲学问题，形而上学的中文名称取自《易经·系辞上传》「形而上者谓之道，形而下者谓之器」一语。1874年，日本启蒙家西周，在《百一新论》中首先用汉文「哲学」来翻译\"philosophy\"一词。\\n\\n英国哲学家罗素对哲学的定义是：\\n\\n胡适在《中国哲学史大纲》中称「凡研究人生切要的问题，从根本上着想，要寻一个根本的解决：这种学问叫做哲学」。\\n\\n虽然哲学源自西方的传统，但许多文明在历史上都存在著一些相似的论题。东亚和南亚的哲学被称之为东方哲学，而北非和中东则因为其和欧洲密切的互动，因此常被视为是西方哲学的一部份。\\n\\n对哲学的主题亦存在许多看法。一些人认为哲学是对问题本身过程的审查；另外一些人则认为实质上存在著哲学必须去回答的哲学命题。\\n\\n\\n古希腊哲学家透过问问题来进行哲学实践，他们所提的问题大概可以归类为三类，这三类问题分别形成了哲学的基础学科——分别是形而上学、伦理学、认识论（或知识论） 。\\n\\n有意思的是，现代哲学上蒙现出\"不要求精确理由\"的哲学论调，如\"本质技巧\"(认定本质不可知)，这种现象将不可知论(世界上终究有人不能理解的存在)的重要性提高了。\\n\\n哲学可以分为很多不同的分支，主要包括形而上学、知识论、伦理学、逻辑学和美学。\\n\\n\\n很多人类社群思考过哲学问题并且互相学习建立了各种哲学流派。\\n\\n东方哲学是通过每个地区的历史时期来组织的。西方哲学一般可以分为三个或更多时期，最重要的是古典哲学、中世纪哲学和近代哲学。\\n\\n印度哲学的历史源远流长，早在吠陀时代已经开始，至公元前6世纪为全盛时期。当时古印度的思想界百花齐放，其中最著名的包括佛教创始人释迦牟尼佛、耆那教创始人笩駄摩那、阿耆多·翅舍钦婆罗、波拘陀·迦旃延、富兰那·迦叶、数论派等。\\n\\n中国哲学的主要部分起源东周时期，当时以诸子百家广为人知，以孔子的儒家、老子的道家、墨子的墨家及晚期的法家为代表，还有一些流派例如农家、阴阳家和名家在之后则名声不显。在秦朝焚书坑儒后除了法家、儒家、道家外其他流派都不再活跃。在当代，中国哲学仍然在亚洲文化扮演一定作用，但是学理上仍在争辩中国哲学是否应归为哲学。\\n\\n古希腊-哲学是西方哲学的一个时期，时间为公元前6世纪[约585]到公元6世纪。它一般被分为三个时期：前苏格拉底时期、柏拉图和亚里士多德的古典希腊时期、和后亚里士多德（或希腊化）时期：有时候会把新柏拉图主义和基督教哲学家们的古典时代晚期加入作为第四个时期。\\n\\n在公元前6世纪的希腊，西方哲学就从古代神话和诗歌中脱颖而出，逐步开始对宇宙的组成以及本源的思考而开始了独立发展。前苏格拉底时期的自然派哲学家们多关注自然界，被认为是西方最早的哲学家，不管他们认识以及解释世界的方式是否正确，但是他们的想法之所以有别于迷信的原因在于，这些哲学家是以理性辅佐证据的方式归纳出自然界的现象。诸如：\\n\\n\\n公元前5世纪中期，普罗泰戈拉和高尔吉亚等所形成的辩士学派将研究的重点由自然转移到人类本身。认为“人才是万物之本”。他们都不相信有真正的存在和真理。普罗泰戈拉认为是非善恶都是相对于人的感觉而言，而高尔吉亚却认为所有的都是同样的假，这是怀疑论的雏形。 \\n\\n公元前6世纪末，以毕达哥拉斯为主的毕达哥拉斯学派所主张的哲学与前述的观点既相近又有不同。罗马古代的历史上记载毕达哥拉斯第一个称自己为哲学家，或者说是爱智慧。他认为“一切都是数字”。其意思就是说一切事物的实质和结构都是它们所包含的数字关系所决定的。他称平均、秩序和调和是宇宙的三大基调，并以音乐的调和说明宇宙的调和。他所在的学派将宇宙总结为十种性质相异的组合：有限与无限、奇与偶、少与多、左与右、男与女、静与动、直与曲、光明与黑暗、善与恶、方与圆。至此之后，数学的本质及其地位，一直都是哲学的主要问题之一，数学不受观察和实验造成的不确定性影响，而且是通过纯粹的思想加以理解的。\\n\\n其中关于变与不变的关系的争论，真实世界与直觉世界的差别，真理与意见的矛盾，导致产生了认识论的问题。\\n\\n在古典希腊时期西方哲学方法的关键特质被建立：依靠诉诸理性和论证，通过一种批判性的方法来接受或建立观点。这包括苏格拉底被称为苏格拉底反诘法或“反驳论证”方法的辩证法，他主要用其来检验例如善良和公平正义的关键道德概念。这种方法将一个问题分解成一系列的疑问，在对疑问的回答中逐步提取想要找到的答案，其极大影响可以从现在使用的科学方法中看出，在科学方法中假说是第一个阶段。\\n\\n苏格拉底没有直接教过人，但之后的柏拉图深受其影响。而其整个哲学思想来源于两大理论：其一，永远不要做坏事；其二，一个内心真正善良且正义的人绝不会做相反之事。他认为真理有其客观性，试图推翻智者们以个人主观感觉为真理的思想。然后提出德的概念，以作为人生行事的方向。对于道德是什么的问题，苏格拉底的回复为“知识即道德。”对于知识是何物的问题，他回答说知识是透过理性而得的概念。苏格拉底开创了认识论和伦理学，如此奠定了他的哲学地位。\\n\\n古典希腊时期的的哲学家中柏拉图和亚里士多德对后世的影响力最大，特别是柏拉图被认为是西方哲学的创始人。哲学家阿尔弗雷德·诺思·怀特黑德评价柏拉图：“欧洲哲学传统最被普遍公认的特点，就是它包含了一系列对柏拉图的注脚。我的意思不是怀疑学者们系统体系的思想是提取自柏拉图的著作。我暗示的是那些他们散落的一般思想的财富。”换言之即使数千年后，人们依旧在试著回答他所提出的问题，这也代表著人们依然为这些问题或是这些问题所延伸的更多问题而感到困惑。\\n\\n毕达哥拉斯的思想对柏拉图产生了显著地影响，并通过柏拉图影响了整个西方哲学。柏拉图和亚里士多德作为最早的古典希腊哲学家批判地引用了其它的一些”智者“，当时这些人在希腊被称为“辩士”并在毕达哥拉斯之前相当普遍。从他们的批判看来，在他们的古典时代一个在更高尚地、纯粹地”爱智慧”（真的哲学家）与那些更早更普遍的旅行教师——经常也通过自己的技艺来赚钱——之间的分水岭之后被建立。\\n\\n亚里士多德死后，整个哲学界陷入了独立时期，称为时期。因为整个社会和政治陷入混乱。这段时期产生了斯多葛学派和伊壁鸠鲁学派，以及怀疑主义派、新柏拉图派和。这些学派的共同特点是伦理化。斯多葛学派主要是顺应自然和自制。伊壁鸠鲁学派则是把快乐作为生活的本质和善的标准。而新柏拉图派和新毕达哥拉斯派都是带有宗教主义的哲学，并逐渐产生融化基督教和希腊哲学于一体的理论，即为后来的基督教哲学。\\n\\n直到公元529年，罗马皇帝查士丁尼一世尼命令关闭雅典的柏拉图学院。称一些余下的学院成员逃入了萨珊王朝首都泰西封。\\n\\n印度哲学是指起源于印度次大陆的哲学思想，包括、、等，这些印度哲学具有一些共同且复杂的起源，都有有关佛法及业的主题，而且都希望达到个人的解放。这些哲学约在西元前一世纪到西元几世纪的时间成形。\\n\\n中世纪哲学指的是西欧和中东在中世纪的哲学体系，其时间范围没有定论，大致上是从基督化的罗马帝国时期至文艺复兴时期。中世纪哲学被部分定义为对古典希腊和希腊化哲学的再发现和进一步发展，另一部分是需要解决神学问题并把亚伯拉罕诸教（伊斯兰教、犹太教和基督教）的教条同世俗知识一同整合并推广。\\n\\n文艺复兴人文学者们排斥中世纪时期，把它当作在希腊罗马的古典时代与古典文化“复兴”之间的一个“过渡”的野蛮时期。然而在中世纪这将近一千年中哲学在欧洲仍取得了长足地发展。认为\"在强度、复杂度还有成就上，可以确信地说哲学在十三世纪的兴盛能与公元前四世纪古希腊哲学的黄金时期媲美。\"\\n\\n这个时代讨论的问题有信仰和理智的关系，神的存在与统一，神学话题和形而上学，关于知识、宇宙和个人的问题。\\n\\n中世纪的哲学家包括基督教学者希波的奥古斯丁、波爱修斯、安瑟伦、、皮埃尔·阿伯拉尔、罗吉尔·培根、圣文德、托马斯·阿奎那、邓斯·司各脱、奥卡姆的威廉和让·布里丹等；犹太哲学家迈蒙尼德和;还有穆斯林哲学家肯迪、法拉比、海什木、伊本·西那、安萨里、伊本·巴哲、、伊本·赫勒敦和伊本·鲁世德等。中世纪的经院哲学传统一直到17世纪仍在活跃，例如和等人物。其中托马斯主义之父阿奎那极大地影响了整个天主教欧洲，他特别强调理性和论证，是最先开始使用亚里士多德形而上学和知识论的著作的新译本的学者之一。他的工作明显远离了统治大部分早期经院哲学的新柏拉图主义和奥古斯丁的思想。\\n\\n从文艺复兴开始，人们的思想开始从清净的僧院走出，来到喧嚣的尘世。从而发展自然，也发展人类自身。从而形成人文主义和自然哲学两股既有联系又有区别的思潮。\\n\\n\"文艺复兴\"是对中世纪到近代之间过渡时期的通称，那时对古典文献的重新学习帮助把哲学界的兴趣从对逻辑学、形而上学和神学领域的钻研转移到包括道德、语言学和神秘主义的更加广泛的研究。对经典和人文艺术例如历史学和文学的研究在基督教世界学术界中享有前所未有的兴趣，这个趋势被称为人文主义，它受到柏拉图主义、希腊怀疑主义和罗马斯多葛主义的影响。人文主义者的哲学兴趣跟随彼特拉克转移到造物主与其美德上，替代了中世纪时对形而上学和逻辑学的兴趣。\\n\\n那时对古典哲学的研究出现了两种新方式。一方面对亚里士多德的研究因为的影响而产生了变化。阿威罗伊亚里士多德主义者和更正统的天主教亚里士多德主义者譬如艾尔伯图斯·麦格努斯和托马斯·阿奎那之间的分歧最终在文艺复兴发展出一种“人文亚里斯多德哲学”，譬如和的思想。另一方面，在一些之前不为西欧所熟知的作品的重发现的帮助下，对柏拉图和新柏拉图主义的研究作为另一个选择变得普遍起来。著名的文艺复兴时期的柏拉图主义者包括库萨的尼古拉，还有之后的马尔西利奥·费奇诺和若望·皮科·德拉·米兰多拉。\\n\\n文艺复兴也重新产生了对反亚里士多德的把自然看作一个有机的、活生生的整体而不取决于神学的理论的兴趣，例如在库萨的尼古拉、尼古拉·哥白尼、焦尔达诺·布鲁诺、还有托马索·康帕内拉的著作中。在自然哲学中这样的运动与对神秘主义、魔法、赫尔墨斯主义还有占星学等兴趣重燃相契合，它们被认为隐藏着收获知识和掌控自然方法的大门。\\n\\n这些新的哲学运动伴随着欧洲宗教和政治的剧变同时出现：宗教改革和封建制的衰落。虽然参与宗教改革的神学家们对哲学没有直接的兴趣，他们打破了神学和知识权威的传统基础。同时还伴随着信仰主义和怀疑主义的复兴，体现在伊拉斯谟，蒙泰涅和等思想家身上。同时，民族国家政治上逐步的中央集权的过程得到了世俗政治哲学的响应，如尼可罗·马基亚维利（常被描述为第一个现代政治思想家，或者是现代政治思想形成的关键点）、托马斯·莫尔、伊拉斯谟、尤斯图斯·利普修斯、让·博丹和胡果·格老秀斯等的著作。\\n\\n先秦诸子之后的两汉经学、魏晋玄学等都是中国哲学的一部份，自唐朝起佛教也开始对哲学产生重要影响；不过中世纪中国哲学最主要的部分是宋明理学的发展。\\n\\n宋明理学反对汉代后开始影响儒学的道教和佛教中的迷信和神秘的元素，是一股倡导更加理性和世俗化儒学的哲学运动。尽管理学遭到道教和佛教徒的批评，理学仍借鉴了它们两个的部分术语和概念。然而和佛教和道教把形而上学看作心灵发展、宗教启示的催化剂并且是不朽的不同，宋明理学把形而上学当作建立一个理性的伦理体系的指导。宋明理学的起源可以追溯到唐朝：韩愈和李翱被视为宋代理学的先驱。宋代理学家周敦颐以道教形而上学理论为框架建立了他的伦理哲学体系，他被看作是宋明理学的创始人。\\n\\n在东亚的其他地方，日本哲学形成于本土的神道信仰和佛教、儒家以及另一些中国哲学和印度哲学学派混合发展。与日本类似，在中巫教的情绪化内容被混合到了从中国传入的理学当中。\\n\\n主条目：近代哲学\\n\\n西方哲学史上的近代早期一般指17世纪和18世纪，其中18世纪常被称为启蒙时代。现代哲学不同于其前身，它和传统权威例如教会、学院、亚里士多德的关系更加独立，出现了对知识基础和形而上学体系建设的新兴趣；和摆脱了自然哲学的近代物理学的出现。从17世纪开始，近代哲学就以认识论为研究重点。由于经验论（经验主义）与唯理论（理性主义）的争论，使物质与精神的关系作为认识论的首要问题突显出来。\\n当时其他的哲学焦点包括精神的天性和其与身体的关系，新的自然科学对诸如自由意志和神的传统上属于神学的话题的影响，和伦理学和政治哲学的世俗基础\\n。这种潮流最早被鲜明地体现在弗兰西斯·培根的被称为用来扩展知识的新的、经验主义的程序，并很快在笛卡儿的机械主义物理学和理性主义的形而上学中建立了具有巨大影响力的形式。培根运用归纳法，第一个提出思维的主体“人”应该主动干涉自然来为人服务。\\n\\n近现代政治哲学的鼻祖托马斯·霍布斯最早将这套方法论系统得应用在政治哲学上，包括\"社会契约\"的近代理论。早期近代哲学的学术经典一般包括笛卡尔、斯宾诺莎、莱布尼茨、洛克、贝克莱、休谟和康德。同时其的其他思想家也对哲学做出了贡献，例如伽利略、皮埃尔·伽桑狄、布莱兹·帕斯卡、马勒伯朗士、艾萨克·牛顿、、孟德斯鸠、、托马斯·里德、让·勒朗·达朗贝尔和亚当·斯密，而让-雅克·卢梭是反启蒙运动的开创性人物。早期近代哲学的大致结束通常被确定为伊曼努尔·康德的试图限定形而上学范围、证明科学知识并用道德和自由来调和两者的体系。\\n\\n理性主义者中勒内·笛卡儿认为物质世界是由数学关系组成的单一体系，他企图将物理学转化为数学。他在其著作中，对整个经院哲学以及在他那个时代流行的教育与哲学体系加以讽刺。其认为“我思故我在”是认识论的无可怀疑之出发点。笛卡尔是割裂精神和物质的二元论者，为了厘清二者关系，他坚定认为在上帝那里，精神和物质是统一的。其理论被称为笛卡尔主义\\n。斯宾诺莎是笛卡尔之后，又一位著名的唯理论者。他的认识论、几何学和机械观都来自于笛卡尔。但他不认同笛卡尔的二元论，认为精神和物质不过是唯一实体的两种属性\\n。莱布尼茨作为唯理论者坚定地维护笛卡尔的学说并反驳约翰·洛克的理论。与笛卡尔不同的是，他认为万物的实体是“单子”，且互相没有关系，而是由于“前定和谐”才共存一体，即存在于神之中。“前定和谐”调和了笛卡尔之二元论和斯宾诺莎之实体双重性。\\n洛克发展了经验论，他不认同笛卡尔的“天赋观念”，提出白板说，他强调人们从感觉中抽象出普遍的概念，认为感觉中的个别东西才是第一位的。不过他基本认同笛卡尔的二元论。贝克莱发展了洛克的哲学理论，提出了“存在就是被感知”。他认为除了感知的主题和被感知的知觉之外，什么也没有。他非常不赞同物质的抽象概念，认为其既无客观实在，也不能存在于人心。大卫·休谟的理论比贝克莱的更进一步，他不仅仅认为物质实体不存在，更认为精神实体不存在。只承认知觉的存在。他还以自己的不可知论和怀疑论认为不存在统一性和普遍性的东西，认定多样性和个别性才是最高原理。\\n\\n经验论与唯理论的争论也包含了唯物主义与唯心主义的争论。在18世纪时，法国的拉美特利公开宣布唯物主义是唯一的，而百科全书的主编德尼·狄德罗也拒绝承认神的存在。\\n\\n另外伏尔泰，孟德斯鸠和其他百科全书派的学者都有涉及政治和伦理领域。他们都认为机械主义才是最终形式——物质是唯一的且处于永恒运动的，精神只是人脑的属性。因此他们认为无机物与有机物不可逾越，人的思维是人感官的结果。不过他们仍然是经验主义者，在因果性上，他们认为只有必然性才是唯一的，这就成为唯心主义的观念。\\n\\n从18世纪中后期开始，直到19世纪初，哲学便进入了近代哲学的总结时期，这就是德国古典哲学时期。有两条线索标志着转折的到来：一、思维与存在的关系更加明确；二、产生了系统辩证法。其代表人物有I.康德、J.G.费希特、F.W.谢林、G.W.F.黑格尔等。\\n\\n康德给哲学带来了三个标志性的创造：\\n\\n\\n他受到休谟的诸多影响，并为西方哲学带来一次革命。他认为哲学的研究核心就是规定理性能做什么以及不能做什么。\\n\\n康德同意休谟的理论并认为，存在一些原则，使得心灵对经验和认识加以组织，而证据皆可以在数学中找到。即是，包含在命题里的要比包含在原是概念的定义要多得多。他使用称之为批判哲学的先验方法，来展现经验的某些范畴和形式都必然地被预先存在于人们一切言谈之中。\\n\\n凭借着他的三部“批判性”的著作，为先验方法作出相应的结构：\\n\\n\\n他还为道德哲学奠定了新基础，且他赋予了自由概念的新意义。因为其影响在现代依旧尚存，其理论被人们称为康德主义。\\n费希特本来承认斯宾诺莎的机械的因果决定论，但后来受到康德的影响，开始认为因果决定论只是表面，其实质为自我不是必然性的奴仆而是独立自由的主体。就此，他建立了主观的思维与客观的存在之统一说。\\n\\n谢林是从费希特理论出发的，但深受斯宾诺莎和文学上浪漫主义的影响，创立了自己的学说。即他认为自然和精神、存在和思维，客体和主体，表面相反，实则统一，是同一个“绝对”的不同发展阶段，这个“绝对”即是万事万物的根源。他认为艺术才是最直观的理性。\\n黑格尔及其理论的出现将西方哲学的推上一个新高度，他创立了西方哲学史上最庞大的客观唯心主义体系，并系统地阐述了辩证法。他的理论和学说对近现代哲学产生了很深远的影响，并被称为黑格尔主义。\\n\\n从黑格尔的思想体系中发展而成的多种哲学运动。其重点就是以历史和逻辑为主，历史方面，它从不同角度理解“凡是合理的就是现实的”；逻辑方面，它有发现其中所说的“真理即整体”。\\n\\n黑格尔认为哲学的重点是放弃分裂，达到统一。他把以前的时代说成是思维与存在、理想与现实分裂，自由与必然，个人与社会、无限与有限、统一性与多样性分裂之时代。\\n\\n他从康德的“心灵的合理性以及在经验中的积极作用”的概念出发，但反对康德的“超越经验世界和‘物自身’的世界”，并认为心灵和世界一样具有相同基础理性结构。他所认为的普遍性不是脱离特殊的抽象普遍，而是包含特殊在内之普遍，即为具体普遍；他所认为的统一也非脱离矛盾、对立的抽象统一，而是包含它们在内的统一，即为对立统一。上述综合在一起即是他的理论：最真实的无所不包的整体即是“绝对精神”，又是对立的统一。\\n\\n他认为，为了达到这个“绝对精神”，需要经过三个阶段，从逻辑、自然到精神，即是从思维到存在，再到两者统一的过程，从而完成他的统一论。\\n\\n就此，社会和历史的现象，便被赋予一种在哲学史上还是崭新的显赫地位。他还将伦理学划归到这个领域，从而在伦理学理论和对思想的理解中提出重要的路线。\\n\\n从19世纪中叶开始，西方哲学就进入现代哲学阶段。因为在19世纪中期，欧洲的工业革命几近完成。\\n\\n现代哲学，特别是19世纪中后期的哲学流派，有叔本华的意志主义，新康德主义，新黑格尔主义，马克思主义。然而此时的哲学与后来的存在主义、现象学等在当代一般归为「欧陆哲学」，与二十世纪以后著重严谨逻辑与语词分析所发展出的「分析哲学」成为风格迥异的两大西方哲学典范。\\n\\n20世纪的西方哲学上主流有两条：\\n\\n现代哲学主要包含以下几种潮流。\\n历程哲学：\\n\\n主流马克思主义：\\n\\n西方马克思主义：\\n\\n革新的黑格尔主义：\\n\\n结构主义：\\n\\n分析哲学：\\n\\n实证主义：\\n\\n新康德主义：\\n\\n逻辑实证主义：\\n\\n语言哲学：\\n\\n现象学：\\n\\n唯物论：\\n\\n新托马斯主义：\\n\\n科学哲学：\\n\\n意志主义：\\n\\n实用主义：\\n\\n存在主义：\\n\\n解释学：\\n\\n唯心主义的各种变体在18世纪晚期至20世纪早期的哲学界相当流行。康德主张的先验唯心主义认为人们对事物的理解是有界限的，因为在客观判断条件下很多事情是办不到的。他在1781年发行的作品《纯粹理性批判》试图调和18世纪两大主要的哲学派别：经验主义和理性主义，并且建立一个研究形而上学的新基础。\\n\\n德国唯心主义最著名的作品是黑格尔于1807年出版的《精神现象学》。黑格尔承认自己的理念不是新的，不过他的目标是完成之前的哲学家们的不完整的体系。黑格尔认为哲学的重点是放弃分裂，达到统一。他把以前的时代说成是思维与存在、理想与现实分裂，自由与必然、个人与社会、无限与有限、统一性与多样性分裂之时代。他从康德的“心灵的合理性以及在经验中的积极作用”的概念出发，但反对康德的“超越经验世界和‘物自身’的世界”，并认为心灵和世界一样具有相同基础理性结构。他所认为的普遍性不是脱离特殊的抽象普遍，而是包含特殊在内之普遍，即为具体普遍；他所认为的统一也非脱离矛盾、对立的抽象统一，而是包含它们在内的统一，即为对立统一。上述综合在一起即是他的理论：最真实的无所不包的整体即是“绝对精神”，又是对立的统一。黑格尔认为需要经过三个阶段来达到这个“绝对精神”，从逻辑、自然到精神，即是从思维到存在，再到两者统一的过程，从而完成他的统一论。他还将伦理学划归到这个领域，从而在伦理学理论和对思想的理解中提出重要的路线。\\n马克思主义哲学是马克思和恩格斯建立的以辩证唯物主义为核心的哲学体系。其认为实践是检验哲学之真理性的最终标准，哲学应伴随着社会、科学技术和文化的发展而不断发展。其主要思想体系在19世纪70年代主要由恩格斯创立，20世纪20年代在苏联形成完整体系——辩证唯物主义和历史唯物主义，这个体系在后来的社会主义国家推动下得以发展。马克思主义哲学宣称自己的理论体系具有科学性，认为哲学可以成为科学的一部分。同时马克思主义哲学认为哲学还具有意识形态的性质。\\n\\n另外马克思主义在政治上也指各种不同的共产主义运动，如由列宁所创立而被斯大林修改的苏联马克思主义，称为马克思列宁主义，为俄国革命以及后来建立的各种共产党之教义。它的旁系包括反斯大林的托洛茨基及其追随者的马克思主义、毛泽东的马克思列宁主义等。\\n实用主义产生于19世纪70年代的现代哲学派别，在20世纪的美国成为一种主流思潮。对法律、政治、教育、社会、宗教和艺术的研究产生了很大的影响。实用主义也试图在理性主义及经验主义找出一条中间道路来，是「经验主义思想方法与人类的比较具有宗教性需要的适当的调和者。」\\n\\n现象学是由德国哲学家胡塞尔在1900年提出的理论，强调对直接直观和经验感知的区分，认为哲学（或至少是现象学）的主要任务是厘清二者之间的关联，并且在直观中获得对本质的认识。现象学是对经验结构与意识结构的哲学性研究。作为一个哲学运动，现象学于二十世纪早期由埃德蒙德·胡塞尔创立，之后被他在德国的哥廷根大学和慕尼黑大学中的一派追随者发展壮大。在此之后现象学传播到法国、美国以及其他地区，并远超出了胡塞尔早期著作的语境。 其他主要哲学家包括海德格(Martin Heidegger), 梅洛—庞蒂(Maurice Merleau-Ponty), 以及列维纳斯(Emmanuel Lévinas)。\\n\\n存在主义是一个哲学的非理性主义思潮，该术语被用在十九世纪晚期到二十世纪的一些哲学家的工作上，尽管他们的学说相差巨大，但他们都相信哲学思考开始于人类主体——而不仅仅是思维主体，而且包括行为、感知、人类个体。存在主义强调个人、独立自主和主观经验，认为人存在的意义是无法经由理性思考而得到答案。在存在主义中，个体的出发点的特征是被称为“存在的态度”，或一种面对显然是一个无意义的或荒谬的世界的迷失和混乱的感觉。很多存在主义者还认为传统的体系和哲学学术无论是内容和风格都过于抽象并远离人类经验。\\n\\n19世纪哲学家克尔凯郭尔和尼采被看作存在主义的先驱，尽管他们没有使用这个术语。然而他们的影响延伸出了存在主义思想。克尔凯郭尔著作主要针对的是黑格尔的唯心主义哲学体系，他认为其忽视或排除了人类的内在主观生命。相反克尔凯郭尔认为\"真理是主观的\"，主张对一个现实的人类来说最重要的问题是处理个人与存在内在关系的问题。克尔凯郭尔作为一个基督徒相信宗教信仰的真相是一个主观问题，而且人应该用热情去深思这个问题。\\n\\n\\n\\n\\n\\n'},\n",
       " {'text1': '文学',\n",
       "  'text2': '文学\\n\\n文学（），在最广泛的意义上，是任何单一的书面作品。\\n\\n更严格地说，文学写作被认为是一种艺术形式，或被认为具有艺术或智力价值的任何单一作品，通常是由于以不同于普通用途的方式部署语言。\\n\\n它的拉丁词根\"literatura\"/\"litteratura\"（本身起源于\"littera\"：\"letter\"或\"handwriting\"）被用来指代所有的书面记录，尽管当代定义将术语扩展到包括口头或唱歌的文本（口头文学）。文学可以根据是虚构作品还是非虚构作品进行分类，也可以根据是韵文还是散文进行分类；可以根据长篇小说、中篇小说、短篇小说等主要形式进一步区分；作品往往根据历史时期或者遵守某些美学特征或期望（艺术类型）进行分类。\\n\\n以语言文字为工具形象化地反映现实的艺术，包括韵文、散文、剧本、小说等，是文化的重要表现形式，以不同的流派表现内心情感和再现一定时期和一定地域的生活。\\n\\n这个概念随著时间的推移而改变了意义：现在它可以扩大到非书面的口头艺术形式，可以与语言或文字本身配合，因此很难就其起源达成一致。\\n\\n印刷技术的发展使得书面作品的分布和扩散成为可能，最终导致了网络文学。\\n\\n文学并不一定是客观的，一名成功的文学家能在自己的文学作品中，展现自己对于文学的主观看法，抒发自己的情绪和感触，但借由尝试建立一个「客观的标准」，有时对能帮助作家了解「读者的感受」以求将内心之情感与艺术表现完整的体现在读者心中。\\n\\n有时也能藉作家主观想法带给社会不同面相去省思现况，例如女性文学的兴起。\\n\\n文学的历史和文明发展有密切的关系。若将文学定义为用文字记录的作品，最早的古代文学作品一般认为是古埃及文学及。古埃及文学中主要的文类（赞美诗、祈祷文及故事）几乎都是以诗的方式写成的，不过虽然可以清楚看出有使用诗歌技巧（poetic devices），但诗歌的韵律不明。最早已知的文学作品是公元前2700年一篇由苏美人创作的《吉尔伽美什史诗》，当中描述英雄主义、友谊、损失及追逐永生。\\n\\n不同的历史时期有著不同特色的文学。古代的文学中有许多有关世界起源及习俗起源内容，也有一些其中有道德及灵性意涵的神话。铁器时代的荷马史诗及以较晚一些的有较多有关作者的资讯，而许多的神话则是用口头传播的方式流传下来。\\n\\n各种文学都可以视为是文字的纪录，文学本身可能是写实或是虚构，但都可以描绘出一些事实，例如主角的动作及言语、作者的写作风格，以及文字后的含义等。这些情节不只是娱乐性的，其中也包括了经济、心理、科学、宗教、政治、文化及社学的相关资讯。在学习历史时，研究及分析当时的文学也是重要的一部份。研究过去的文学可以看到不同历史时期时，其社会和社会规范的演变，甚至于也可助于了解现今的文学，因为其中常常引用古希腊神话、宗教典藉及相关文献的资料。人们不止可以从各主题相关的文学中看到该主题随著历史的演进（例如从经济史的书或介绍科学及演化的书），甚至连科幻小说中都可看到类似的内容。作者常常在其作品中加入一些历史的内容，例如拜伦勋爵在《Childe Harold’s Pilgrimage: Canto I》中借由主角Childe Harold提到西班牙文及法文，也提到作者的一些想法。借由文学人们可以继续的发现有关历史的新资讯，这个从各个学科领域都有和文学相关的子领域可以看出。当人们将资讯用文字的方式纪录下来，就比较容易从这一代流传到下一代，留下来的资讯会越来越多。从这些资料，人们可以研究文学、提升想法、扩展知识、也可以开始像医学或是贸易等专业领域的研究。而随著现代人们学习内容的增加及拓展，文学也会有一些不同，成为以后人们研究的基础。\\n\\n许多古文明都有其对哲学或是相关观点的文学，像是古中国、古印度、波斯时及希腊罗马古典时代的作品。许多古代的作品，就算是叙事的形式，都还是有道德或是教诲上的目的，像梵语的《五卷书》或是奥维德的《变形记》，后来戏剧及讽刺作品的受众也变多，因此也开始有类似性质的文学创作。抒情诗常常是贵族圈的特性产物，特别在东亚，许多歌曲被贵族收集，成为诗歌。\\n\\n浪漫主义的异常特质在中世纪绽放。同时，理性时代造就了民族主义史诗与哲学短文。浪漫主义强调通俗的文学及情感的投入，慢慢被寻求真实的现实主义与自然主义文学所取代。到了20世纪，象征主义抬头，探索角色的描述和发展。\\n\\n在很长一段时间，中国的文学与史学和神话并无明显的界限，最早的文学是对历史和神话的记录。但纯粹的文学早在周时就已出现，例如《诗经》。中国古代的文学主要著重在哲学、史学史、军事学、农业及韵文。中国发明了造纸术及雕版印刷，也是世界上第一个。中国的许多哲学思想是起源自春秋战国时的诸子百家，其中最重要的有儒家、道家、墨家及法家，而军事学书籍（如孙子兵法）也是在春秋战国时开始出现。中国历史文学则从尚书、春秋、战国策、史记等一直延续下来，而且有很详细的资料记录。\\n\\n中国的文学成就最大的是诗歌，从《离骚》到唐代律诗，诗歌一直对中国文坛有着巨大的影响。后来诗、词、曲、小说等文学形式分别在唐、宋、元、明清达到高峰。民国时期由胡适和陈独秀推动的新文学运动，认为作品不应只讲求形式，应注重内容的充实、表达及情感，也推动白话文学。民国时期，武侠小说风靡海内外，成为当时最受欢迎的通俗小说。\\n\\n中华人民共和国时期，在文化大革命后，出现相关的反思文学及伤痕文学，也有一批白话文诗人进行大量创作，也取代古诗成为当时最欢迎的诗歌作品。后来网路文学兴起，成为受欢迎的商业作品。\\n\\n中华民国在撤退台湾后，在50及60年代出现了以四大抗战小说为代表的战斗文艺小说，都是以抗战时期为背景，后来又有反共文学的出现，而60年代开始，以琼瑶为代表的言情小说也开始行。70年代起逐渐开始有对于台湾社会研究的新现代文学，以及强调乡土的乡土写实文学，1990年后也开始了网路文学的兴起。\\n\\n\\n中国古典文学分为诗和文，文又分为韵文和散文，中国的抒情诗和文言文最早而比较发达。\\n\\n文学一般分为小说、散文、诗歌、剧本，并称为四大文学体裁；\\n\\n\\n\\n剧本是另一种古老的文学形式，主要通过不同角色之间的对话来表达作者的思想和感情。剧本可以用于舞台的表演，也可以阅读。像元曲、京剧、昆剧都属于这个部份。西方的戏剧许多都伴随著音乐和舞蹈，例如歌剧及音乐剧，古希腊戏剧是目前已知最早期的西方戏剧，有悲剧、喜剧、悲喜剧等。\\n\\n\\n\\n有许多的文学奖，颁发给优秀的作家，表扬其文学的成就。因为文学的范围很广，许多文学奖项会依风格、文学类型、语言、国籍及其他特性（例如新进作家或是等）再做分类。\\n\\n诺贝尔文学奖是依诺贝尔在1895年的遗嘱所成立的奖项，是诺贝尔奖中的一项，一般是因为作者的整体作品而获奖，而非著重特定的作品。其他不分国籍的奖项有：纽斯塔特国际文学奖、布克国际奖及卡夫卡奖。\\n\\n\\n是文学创作者应用在文学中，制造特别效果的方式。文学技巧的范围很广，包括作品是否要用第一人称或是其他人称、用传统的线性叙事或是、或是文类选择都包括在内。这可以让读者感受到一些熟悉的结构及架构，例如传统犯罪小说，不过有些作者会特别选择一些文学技巧来让读者有意外的感受。\\n\\n文学技巧的使用也可能会产生新的文类，就像塞缪尔·理查森写的早期现代小说《》一様。《Pamela》是用许多的信件组成，称为「书信体技巧」（epistolary technique）。因此《Pamela》让大家再次注意到，一个以往曾出现，但没有这么受注意的文类。\\n\\n文学技巧和文学手段（literary device）不同，有点类似军事战略和军事战术之间的关系。文学手段是在叙述中用的特殊结构，像是隐喻、明喻、省略、叙事及托寓等，甚至单纯的谐音都可以作为文学手段。也可以视为是文学手段，例如意识流叙事。\\n\\n文学批评是指文学批评者对其他人作品的评论和评估，有时也会用来改进及提升文学作品。也可以对作者带来类似的作用。有许多不同种类的文学批评，背后会有其理论基础，不同种类的文学批评可以评论文学作品的各个部份或是各个层面。\\n\\n\\n\\n\\n\\n\\n\\n\\n'},\n",
       " {'text1': '历史',\n",
       "  'text2': '历史\\n\\n历史（现代汉语词汇，古典文言文称之为史），指人类社会过去的事件和行动，以及对这些事件行为有系统的记录、诠释和研究。历史可提供今人理解过去，作为未来行事的参考依据，与伦理、哲学和艺术同属人类精神文明的重要成果。历史的第二个含义，即对过去事件的记录和研究，又称历史学”，或简称“史学”。隶属于历史学或与其密切相关的学科有年代学、编纂学、家谱学、古文字学、计量历史学、考古学、社会学和新闻学等，参见历史学。记录和研究历史的人称为历史学家，简称“史学家”，中国古代称为史官。记录历史的书籍称为史书，如《史记》、《汉书》等，粗分为「官修」与「民载」两类。\\n\\n广义的历史，泛指一切事物的发展过程，包括自然史和社会史。不一定同人类社会发生联系。在哲学上，这种含义下的历史称为历史本体，例如宇宙历史、地球历史、世界历史、中国历史等等。通常仅指人类社会的发展过程，它是史学研究之对象；一般说来，关于历史的记述和阐释，也称为历史。而狭义的历史则必须以文字记录为基础，即文字出现之后的历史才算历史，在此之前的历史被称为史前史。又可以称为人类史或社会史，而脱离人类社会的过去事件称为自然史。一般来说，历史学仅仅研究前者，即社会史。\\n\\n“历史”的含义在中文中最早仅用“史”一字来代表。甲骨文中“史”字与“事”相似，指事件。许慎《说文解字》说：“史，记事者也；从又持中，中，正也。”便指出“史”的本意即记事者，是一人执“中”之象。近人金静庵说：“保藏之档案谓之中，持中之人谓之史。一指书言，一指人言。”，由此引申，则代表被史官被纪录的事，换句话说，即所有被文字纪录的过去事情。研究史的学问，称史学。\\n\\n“历史”一词出现较晚，在《三国志》裴松之注中，首次提到历史二字。《南齐书》中也提到这个名词，是历代史书之意。明代嘉靖年间李廷机与叶向高编辑《历史大方通鉴》，是中国第一本以历史二字为名的书，袁了凡为此书写的〈历史纲鉴补引〉，解释历史是指诸史，也就是历代史书之意。\\n\\n1895年，礼部侍郎于式枚在奏章中，提到历史这个名词，将它作为历史事件之意。1896年，皮锡瑞《经学历史》出版，其中的历史二字，也明确将它定位为历史事件之意。因此，在1890年代前后，历史这个名词在中国开始被明确当作历史事件来使用。\\n\\n明治维新后，日本学者为翻译，译为历史二字，使其成为对应词。1870年代成为流行用语。\\n\\n1902年，光绪皇帝接受吏部尚书张百熙建议，颁布〈钦定学堂章程〉，其中寻常小学课目中，有史学、舆地二项。张百熙派吴汝纶赴日本考察教育后，1903年，负责教育改革的张百熙、张之洞、荣庆向皇帝建议重订学堂章程。在重订章程后，寻常小学依日本语改称初等小学，而史学、舆地二科，则改称历史、地理。之后，历史这个名词在中国流行。\\n\\n在欧美，多数语言的“历史”一词源出自（Historia），原义为“调查、探究、知识”，古希腊作家希罗多德的《历史》（Historia）一书以此为名。\\n\\n对于历史的含义和性质，有很多种不同的诠释，以下列举其中一些。\\n\\n历史并不是归类于人文科学或社会科学中，而是其间的桥梁，合并了两大领域的研究方法。一般来说，史学家通过研究各种书面文字但并不局限于此，努力并尝试解答和历史有关的问题。历史知识的原始资料分为三种：文字记载的、口头流传的、保留下来的历史遗迹，通常历史学家会综合三种方法进行研究，而文字记载经常被作为强调的重点，因为它普遍纪录了发展的时间。这种强调引申出了一个新领域，史前史，也可称为史前学，研究的是没有书面纪录的那一个时期。由于世界各地文字出现的时间各不相同，所以史前史和历史的主要区别是根据具体的论题而决定。学者们为了易于研究，根据过去人类的范围将其划分为不同的阶段。划分过去的方法繁多，包括按年代分类，按文化习俗分类，按不同主题分类。这三种分类经常会有重叠，比如“阿根廷的劳工运动的演变，1930-1945”。\\n\\n尽管历史研究倾向于一些专门的地点、时间和主题，历史学家也同时会关心其他普通的一些内容。而对于其他人来说，历史已经成为一个非常普通的词语，就是研究过去人类的所有事情，甚至于现在更兴起了一门所谓的广义历史。过去研究历史都是为了应用或者理论的目的，而现在还多了一条：那就是对人类过往的好奇。\\n\\n以历史为认识对象所形成的一门学问，叫史学或历史学，也可以用“历史”一词代表。历史学的本质其实是把实际发生的事件转换成以意念和文字形式存在的历史的过程和方法。关于历史学的目的和方法的研究探讨，在西方属于历史哲学的范畴，历史哲学的出现和发展，意味着历史学从单纯的历史纪录发展成为对历史的解释和对历史规律的探求阶段。一开始，历史哲学仅仅关心如何改进历史研究的方法，但认为被研究和记录的历史就是真实的历史。在新康德主义和新黑格尔主义的影响下，人们对自身的认识过程有了重新的理解，哲学家开始重新定义历史学。意大利哲学家克罗齐提出“一切真历史都是当代史”的命题，认为往事只有在当代人生活中发挥作用才成为历史，否则是“死的历史”，即编年史。因此，同样的历史在不同的时期会被不断的改写。英国哲学家柯林武德又进一步认为“一切历史都是思想史”，即历史是历史学家思想的反映，不仅因时代而异，也因人而异。而唯物主义的历史观认为历史事件是客观存在的，历史则是历史学家主观对客观的历史事件的认识。由于人主观的局限性，对客观的历史事件的认识是有限的，主观的认识不能完全符合客观的历史，因此只有不断改进逐渐逼近，这一过程同自然科学的过程一致。这种历史学称为“历史科学”。\\n\\n世界历史是世界各地人类过去经验的总和，而且这些经验主要是透过文字的方式保存下来。相对而言，史前是指一地区已有人类产生，但还没有出现文字的时代。借由研究当时的绘画、素描、雕刻或其他工件，可以在没有文字记录的情形下得到一些当时的资讯。自从二十世纪起，研究者开始重视史前的研究，以免历史研究隐性的排除一些特定的文明，例如撒哈拉以南非洲及前哥伦布时期的美洲。在西方的历史学家不成比例的专注在西方世界的研究。1961年时，英国历史学家爱德华·霍列特·卡尔认为：\\n此定义下的历史，也包括一些当时没有文字记录，但对历史有强烈兴趣的民族，像在和欧洲人接触之前的澳大利亚原住民及纽西兰的毛利人，虽没有文字记录，但用口传历史的方式将历史传给下一代。\\n\\n\\n中国是世界上书载历史的传承最完备的国家，其对历史的记录不仅时间长，而且内容精确详细。中国历史自传说中的黄帝以来已经有4千多年，而自西周共和时代（前841年）以来历史记录精确到年，自鲁隐公元年（前722年）以来则精确到月日。中国的历史记录也被称为史或史书，分为编年体，纪传体，纪事本末体等不同体裁。与西方文明中的历史学不占据主流地位相反，中国将史列为四种基本学科分类“经、史、子、集”之一（清纪晓岚等，《四库全书》）。魁奈说：“历史学是中国人一直以其无与匹伦的热情予以研习的一门学问。没有什么国家如此审慎地撰写自己的编年史，也没有什么国家这样悉心地保存自己的历史典籍。”\\n\\n原始社会中人类没有文字，只能通过诸如结绳记事和口传等方法作记录，一些历史的痕迹通过“传说”保存了下来，例如中国上古传说“黄帝战蚩尤”、“女娲补天”、“大禹治水”等。国家出现后，则开始有掌管祭祀的“巫”，他们同时兼任记录时事、起草公文和掌管文书等相关职能，可以说是最早的史官。之后随着国家职能的不断演进与发展，出现了职能独立的史官，专门记录历史事件，掌管典籍。在这个时期，中国出现了世界上最早的史书《尚书》，内容是历代政治文件汇编，并无特定的历史记录体裁。从西周共和元年（公元前841年）起，中国有了按年记载的编年史，从此有了连续不断的历史纪录，而且差不多每年都有史可查。这在世界各国范围内也是极其罕见的。春秋战国时期的史学家如孔子（编订《春秋》）和左丘明（著《左传》）等重视人类社会活动，从而使历史基本摆脱了神学和宗教的影响。\\n\\n西汉时司马迁撰写了《史记》，创建了纪传体的历史记录体裁，《史记》的规模在当时世界范围内是空前的。之后东汉班固著《汉书》，延续发展了《史记》的体例，是中国第一部纪传体断代史。这两部历史著作，奠定了中国古典史学的基础，后来的历史学家沿用《史记》和《汉书》的体裁，将各个朝代的历史汇编成书，组成了“二十四史”。除断代史之外，唐宋期间中国还出现了通史，如唐末杜佑的《通典》，宋司马光的《资治通鉴》，其中《资治通鉴》是叙事长达一千三百六十二年的编年体通史，是中国史学史上的奇葩。\\n\\n西方的历史学开始于公元前5世纪，古希腊作家希罗多德在《历史》（又名《希波战争史》）一书中记录了希腊与波斯之间的希波战争，历史从此自神话和文学中脱离出来成为独立的学科。希罗多德也因此被罗马哲学家西塞罗称为“史学之父”。但希罗多德的记录中真实事件与虚构事件混杂，并不是纯粹的历史。20多年后古希腊人修昔底德所著的《伯罗奔尼撒战争史》治学态度严谨，历史记载翔实，才是西方第一部“信史”。前2世纪，希腊历史学家波里比阿在《通史》（又名《罗马史》）中记录了前218年至前146年73年间罗马帝国周围地中海沿岸各国、各民族的历史，是第一部“世界”通史。\\n\\n历史哲学是哲学的一个分支，主要考虑人类历史的最终意义。更进一步的，它考虑人类历史的可能的目的论的结局。换句话说，它追问人类历史的过程中是否存在着一个设计，目的，指导原则或是定局。\\n\\n史学方法由历史学家在使用第一手资料和其他证据来研究并书写历史时所遵循的技巧和原则。\\n\\n\\n\\n\\n\\n'},\n",
       " {'text1': '计算机科学',\n",
       "  'text2': '计算机科学\\n\\n计算机科学（，有时缩写为）是系统性研究信息与计算的理论基础以及它们在计算机系统中如何与应用的实用技术的学科。 它通常被形容为对那些创造、描述以及转换信息的算法处理的系统研究。计算机科学包含很多分支领域；有些强调特定结果的计算，比如计算机图形学；而有些是探讨计算问题的性质，比如计算复杂性理论；还有一些领域专注于怎样实现计算，比如程式语言理论是研究描述计算的方法，而程式设计是应用特定的程式语言解决特定的计算问题，人机交互则是专注于怎样使计算机和计算变得有用、好用，以及随时随地为人所用。\\n\\n有时公众会误以为计算机科学就是解决计算机问题的事业（比如信息技术），或者只是与使用计算机的经验有关，如玩游戏、上网或者文字处理。其实计算机科学所关注的，不仅仅是去理解实现类似游戏、浏览器这些软件的程序的性质，更要通过现有的知识创造新的程序或者改进已有的程序。\\n\\n尽管计算机科学（computer science）的名字里包含计算机这几个字，但实际上计算机科学相当数量的领域都不涉及计算机本身的研究。因此，一些新的名字被提议出来。某些重点大学的院系倾向于术语\"计算科学\"（computing science），以精确强调两者之间的不同。丹麦科学家Peter Naur建议使用术语\"datalogy\"，以反映这一事实，即科学学科是围绕着数据和数据处理，而不一定要涉及计算机。第一个使用这个术语的科学机构是哥本哈根大学Datalogy学院，该学院成立于1969年，Peter Naur便是第一任教授。这个术语主要被用于北欧国家。同时，在计算技术发展初期，《ACM通讯》建议了一些针对计算领域从业人员的术语：turingineer，turologist，flow-charts-man，applied meta-mathematician及applied epistemologist。 三个月后在同样的期刊上，\"comptologist\"被提出，第二年又变成了\"hypologist\"。 术语\"computics\"也曾经被提议过。在欧洲大陆，起源于信息（information）和数学或者自动（automatic）的名字比起源于计算机或者计算（computation）更常见，如informatique（法语），Informatik（德语），informatika（斯拉夫语族）。\\n\\n著名计算机科学家Edsger Dijkstra曾经指出：“计算机科学并不只是关于计算机，就像天文学并不只是关于望远镜一样。”（\"Computer science is no more about computers than astronomy is about telescopes.\"）设计、部署计算机和计算机系统通常被认为是非计算机科学学科的领域。例如，研究计算机硬件被看作是计算机工程的一部分，而对于商业计算机系统的研究和部署被称为信息技术或者信息系统。然而，现如今也越来越多地融合了各类计算机相关学科的思想。计算机科学研究也经常与其它学科交叉，比如心理学，认知科学，语言学，数学，物理学，统计学和经济学。\\n\\n计算机科学被认为比其它科学学科与数学的联系更加密切，一些观察者说计算就是一门数学科学。 早期计算机科学受数学研究成果的影响很大，如Kurt Gödel和Alan Turing，这两个领域在某些学科，例如数理逻辑、范畴论、域理论和代数，也不断有有益的思想交流。\\n\\n早期计算机科学建立的基础得追溯到最近电子计算机的发明。那些计算固定数值任务的机器，比如算盘，自古希腊时期即已存在。Wilhelm Schickard在1623年设计了世界上第一台机械计算器，但没有完成它的建造。布莱兹·帕斯卡在1642年设计并且建造了世界上第一台可以工作的机械计算器Pascaline。埃达·洛夫莱斯协助查尔斯·巴贝奇在维多利亚时代设计了差分机。1900年左右，打孔机问世。然而以上这些机器都局限在只能完成单个任务，或者充其量是所有可能任务的子集。\\n\\n到了20世纪40年代，随着更新更强大的计算机器被发明，术语“计算机”开始用于指代那些机器而不是它们的祖先。计算机的概念变得更加清晰，它不仅仅用于数学运算，总的来说计算机科学的领域也扩展到了对于计算的研究。20世纪50年代至20世纪60年代早期，计算机科学开始被确立为不同种类的学术学科。 世界上第一个计算机科学学位点由普渡大学在1962年设立。随着实用计算机的出现，很多计算的应用都以它们自己的方式逐渐转变成了研究的不同领域。\\n\\n虽然最初很多人并不相信计算机可能成为科学研究的领域，但是随后的50年里也逐渐被学术界认可。IBM公司是那段时期计算机科学革命的参与者之一。在那段探索时期，IBM（International Business Machines的缩写）发布的IBM 704以及之后的IBM 709计算机被广泛使用。“不过，使用IBM电脑工作仍然是一件很沮丧的事情。如果你弄错了一条指令中的一个字母，程序将会崩溃，而你也得从头再来。”20世纪50年代后期，计算机科学学科还在发展阶段，这种问题在当时是一件很常见的事情。\\n\\n随着时间的推移，计算机科学技术在可用性和有效性上都有显著提升。现代社会见证了计算机从仅仅由专业人士使用到被广大用户接受的重大转变。最初，计算机非常昂贵，要有效利用它们，某种程度上必须得由专业的计算机操作员来完成。然而，随着计算机变得普及和低廉，已经几乎不需要专人的协助，虽然某些时候援助依旧存在。\\n\\n虽然计算机科学被认定为正式学术学科的历史很短暂，但仍对科学和社会作出了很多基础贡献。包括：\\n\\n提出计算机科学可以分成三个领域：数学、工程学、科学。Amnon H. Eden提议了三种范式应用于计算机科学的各个领域：\\n\\n作为一个学科，计算机科学涵盖了从算法的理论研究和计算的极限，到如何通过硬件和软件实现计算系统。 CSAB（以前被叫做\"Computing Sciences Accreditation Board\"），由Association for Computing Machinery（ACM）和（IEEE-CS）的代表组成，确立了计算机科学学科的4个主要领域：\"计算理论\"，\"算法与数据结构\"，\"编程方法与编程语言\"，以及\"计算机组成与架构\"。CSAB还确立了其它一些重要领域，如软件工程，人工智能，计算机网络与通信，数据库系统，并行计算，分布式计算，人机交互，计算机图形学，操作系统，以及数值和符号计算。\\n\\n广义的理论计算机科学包括经典的计算理论和其它专注于更抽象、逻辑与数学方面的计算。\\n\\n算法指定义良好的计算过程，它取一个或一组值作为输入，经过一系列定义好的计算过程，得到一个或一组输出。算法是计算机科学研究的一个重要领域，也是许多其他计算机科学技术的基础。算法主要包括数据结构、计算几何、图论等。除此之外，算法还包括许多杂项，如模式匹配、部分数论等。\\n\\n按照Peter J. Denning的说法，计算机科学的最根本问题是“什么能够被有效地自动化？”计算理论的研究就是专注于回答这个根本问题，关于什么能够被计算，去实施这些计算又需要用到多少资源。为了试图回答第一个问题，递归论检验在多种理论计算模型中哪个计算问题是可解的。而计算复杂性理论则被用于回答第二个问题，研究解决一个不同目的的计算问题的时间与空间消耗。\\n\\n著名的“P=NP?”问题，千禧年大奖难题之一，是计算理论的一个。\\n\\n信息论与信息量化相关，由克劳德·香农创建，用于寻找信号处理操作的根本极限，比如压缩数据和可靠的数据存储与通讯。编码理论是对编码以及它们适用的特定应用性质的研究。编码（code）被用于数据压缩，密码学，前向纠错，近期也被用于网络编码。研究编码的目的在于设计更高效、可靠的数据传输方法。\\n\\n编程语言理论是计算机科学的一个分支，主要处理编程语言的设计、实现、分析、描述和分类，以及它们的个体特性。它属于计算机科学学科，既受影响于也影响着数学、软件工程和语言学。它是公认的计算机科学分支，同时也是活跃的研究领域，研究成果被发表在众多学术期刊，计算机科学以及工程出版物。\\n\\n形式化方法是一种特别的基于数学的技术，用于软件和硬件系统的形式规范、开发以及验证。在软件和硬件设计方面，形式化方法的使用动机，如同其它工程学科，是通过适当的数学分析便有助于设计的可靠性和健壮性的期望。但是，使用形式化方法会带来很高的成本，意味着它们通常只用于高可靠性系统，这种系统中安全或保安（security）是最重要的。对于形式化方法的最佳形容是各种理论计算机科学基础种类的应用，特别是计算机逻辑演算，形式语言，自动机理论和形式语义学，此外还有类型系统、代数数据类型，以及软件和硬件规范和验证中的一些问题。\\n\\n计算机系统结构，或者数字计算机组织，是一个计算机系统的概念设计和根本运作结构。它主要侧重于CPU的内部执行和内存访问地址。这个领域经常涉及计算机工程和电子工程学科，选择和互连硬件组件以创造满足功能、性能和成本目标的计算机。\\n\\n操作系统是管理电脑硬体与软体资源的电脑程式，同时也是电脑系统的核心与基石。作业系统需要处理如管理与配置记忆体、决定系统资源供需的优先次序、控制输入与输出装置、操作网路与管理档案系统等基本事务。作业系统也提供一个让使用者与各电脑设备互动的操作介面。\\n\\n并发性（concurrency）是系统的一种性质，这类系统可以同时执行多个可能互相交互的计算。一些数学模型，如Petri网、进程演算和PRAM模型，被建立以用于通用并发计算。分布式系统将并发性的思想扩展到了多台由网络连接的计算机。同一分布式系统中的计算机拥有自己的私有内存，它们之间经常交换信息以达到一个共同的目的。\\n\\n计算机网络是管理遍及全球的计算机连接成的网络的计算机科学分支。\\n\\n计算机安全是计算机技术的一个分支，其目标包括保护信息免受未经授权的访问、中断和修改，同时为系统的预期用户保持系统的可访问性和可用性。密码学是对于隐藏（加密）和破译（解密）信息的实践与研究。现代密码学主要跟计算机科学相关，很多加密和解密算法都是基于它们的计算复杂性。\\n\\n数据库是为了更容易地组织、存储和检索大量数据。数据库由数据库管理系统管理，通过数据模型和查询语言来存储、创建、维护和搜索数据。\\n\\n计算机图形学是对于数字视觉内容的研究，涉及图像数据的合成和操作。它跟计算机科学的许多其它领域密切相关，包括计算机视觉、图像处理、计算几何与可视化，同时也被大量运用在特效和电子游戏。\\n\\n科学计算（或者计算科学）是关注构建数学模型和量化分析技术的研究领域，同时通过计算机分析和解决科学问题。在实际使用中，它通常是计算机模拟和计算等形式在各个科学学科问题中的应用。\\n\\n资料探勘也就是将人类在过去的历史当中所收集的资料，加以汇集起来，再将这些资料结合机器学习，来达到分析、管理、前瞻等能力。\\n\\n这个计算机科学分支旨在创造可以解决计算问题，以及像动物和人类一样思考与交流的人造系统。无论是在理论还是应用上，都要求研究者在多个学科领域具备细致的、综合的专长，比如应用数学，逻辑，符号学，电机工程学，精神哲学，神经生理学和社会智力，用于推动智能研究领域，或者被应用到其它需要计算理解与建模的学科领域，如金融或是物理科学。人工智能领域开始变得正式源于Alan Turing这位人工智能先驱提出了图灵试验，以回答这样一个终极问题：“计算机能够思考吗？”\\n\\n机器学习是人工智慧的其中一个分支，让机器可以自动学习、从巨量资料中找到规则，进而有能力做出预测。人工智慧让过去只能透过人类或动物智慧解决的问题也能透过电脑系统迎刃而解；机器人是自动执行工作的机器装置，而人工智慧则可以让机器人快速、精准处理大量资料。简单来说，机器人像是人的「身躯」，人工智慧则是人的「脑」。\\n\\n软件工程是对于设计、实现和修改软件的研究，以确保软件的高质量、适中的价格、可维护性，以及能够快速构建。它是一个系统的软件设计方法，涉及工程实践到软件的应用。\\n\\n计算机科学和软件工程的关系是一个有争议的话题，随后关于什么是“软件工程”，计算机科学又该如何定义的争论使得情况更加混乱。David Parnas从其它工程和科学学科之间的关系得到启示，宣称计算机科学的主要重点总的来说是研究计算的性质，而软件工程的主要重点是具体的计算设计，以达到实用的目的，这样便构成了两个独立但又互补的学科。\\n\\n主要进行计算和算法推理的研究。其中包括计算理论、算法分析、形式化方法、并行理论、数据库、计算机图形学以及系统分析等。通常也教授程序设计，但仅仅将它看作是支持计算机科学其它领域的媒介，而不是高级研究的重心。\\n\\n的计算机科学课程则主要侧重于训练高级编程，而不是算法和计算理论。这些课程着重教授那些对于软件工业很重要的技能。像这样的计算机编程过程通常被叫做软件工程。\\n\\n然而，尽管计算机科学专业日益推动着美国经济，但是计算机科学教育依然不存在大多数美国K-12课程中。2010年10月由ACM和计算机科学教师协会（CSTA）共同发表了一篇名为“Running on Empty: The Failure to Teach K-12 Computer Science in the Digital Age”的报告，文中揭示了仅有14个州通过了有意义的高中计算机科学教育标准。同时，仅有9个州将高中计算机科学课程算作毕业要求的核心学科。配合“Running on Empty”这篇文章，一个新的无党派宣传联盟：Computing in the Core（CinC）被建立，以影响联邦和政府政策，比如Computer Science Education Act要求政府拨款以制定计划完善计算机科学教育及支持计算机科学教师。\\n\\n在中国，“计算机科学”或“计算机科学与技术”是工科（一级门类）下的二级专业。一般可细分为三级专业：\\n\\n\\n\\n\\n\\n'}]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.corpus_lines\n",
    "train_dataset.lines[: 5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 1000, 384])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "self_positional_enc.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "optimizer: Adam (\n",
      "Parameter Group 0\n",
      "    amsgrad: False\n",
      "    betas: (0.9, 0.999)\n",
      "    capturable: False\n",
      "    differentiable: False\n",
      "    eps: 1e-08\n",
      "    foreach: None\n",
      "    fused: None\n",
      "    lr: 2e-06\n",
      "    maximize: False\n",
      "    weight_decay: 0\n",
      ")\n",
      "Total Parameters: 23425444\n"
     ]
    }
   ],
   "source": [
    "print(\"optimizer:\", optimizer)\n",
    "\n",
    "print(\"Total Parameters:\", sum([p.nelement() for p in bert_model.parameters()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_most_recent_state_dict(dir_path):\n",
    "    dic_lis = [i for i in os.listdir(dir_path)]\n",
    "    if len(dic_lis) == 0:\n",
    "        raise FileNotFoundError(\"can not find any state dict in {}!\".format(dir_path))\n",
    "    dic_lis = [i for i in dic_lis if \"model\" in i]\n",
    "    dic_lis = sorted(dic_lis, key=lambda k: int(k.split(\".\")[-1]))\n",
    "    return dir_path + \"/\" + dic_lis[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoint[\"model_state_dict\"] OrderedDict([('bert.embeddings.word_embeddings.weight', tensor([[ 0.0112,  0.1011,  0.0444,  ...,  0.0314, -0.0032, -0.1670],\n",
      "        [-0.0045, -0.0391,  0.0037,  ...,  0.0233, -0.0054, -0.0109],\n",
      "        [-0.0679, -0.0397, -0.0364,  ..., -0.0282, -0.0413, -0.0045],\n",
      "        ...,\n",
      "        [ 0.0034, -0.0647, -0.0567,  ..., -0.0489,  0.0013, -0.1270],\n",
      "        [ 0.0291,  0.0950,  0.0634,  ...,  0.0488, -0.0124, -0.1808],\n",
      "        [ 0.0464,  0.0704,  0.0567,  ...,  0.0335,  0.0045, -0.1737]])), ('bert.embeddings.token_type_embeddings.weight', tensor([[ 0.0364,  0.0524,  0.0243,  ..., -0.0368,  0.0375,  0.0038],\n",
      "        [ 0.0327,  0.0459,  0.0188,  ..., -0.0506,  0.0252, -0.0114],\n",
      "        [ 0.0040,  0.0988,  0.0151,  ..., -0.0302,  0.0228,  0.0075],\n",
      "        ...,\n",
      "        [-0.0442,  0.0838,  0.0446,  ...,  0.0609,  0.0587, -0.0699],\n",
      "        [ 0.0429, -0.0784,  0.1099,  ...,  0.0763, -0.0340, -0.1117],\n",
      "        [ 0.0616, -0.0299,  0.0411,  ...,  0.0114, -0.0003,  0.0240]])), ('bert.embeddings.LayerNorm.weight', tensor([1.0275, 1.2026, 1.2092, 1.4398, 1.2014, 1.1223, 1.1936, 1.1698, 1.2504,\n",
      "        1.1670, 1.0946, 1.3940, 1.2938, 1.1391, 1.1378, 1.0212, 0.5480, 1.1130,\n",
      "        1.0899, 1.0682, 1.0721, 1.0662, 0.9285, 1.1541, 1.3891, 1.1252, 1.1271,\n",
      "        1.0311, 0.8423, 0.8541, 0.7478, 0.8294, 0.9441, 0.9278, 0.8740, 0.8934,\n",
      "        0.9136, 0.9088, 0.9553, 0.9360, 0.8806, 0.6722, 0.8308, 0.8239, 0.7308,\n",
      "        0.6734, 0.7538, 0.7201, 0.7448, 0.8269, 0.7882, 0.8167, 0.9235, 0.6934,\n",
      "        0.8336, 0.7797, 0.8024, 0.7747, 0.8045, 0.7069, 0.7545, 0.7114, 0.6991,\n",
      "        0.6924, 0.7559, 0.6707, 0.6854, 0.7641, 0.7657, 0.7674, 0.7898, 0.7205,\n",
      "        0.7759, 0.7260, 0.7117, 0.7894, 0.7743, 0.8107, 0.7815, 0.6592, 0.7101,\n",
      "        0.5524, 0.9598, 0.8204, 0.9608, 0.6495, 0.7472, 0.7524, 0.8147, 0.7328,\n",
      "        0.8968, 0.8612, 0.8923, 0.7473, 0.9009, 0.8711, 0.9064, 0.9309, 0.9410,\n",
      "        0.9512, 0.8541, 0.3008, 0.9227, 0.8992, 0.9263, 0.3648, 0.9179, 0.8744,\n",
      "        0.9750, 0.8894, 0.8881, 0.9544, 1.0116, 0.9579, 0.8847, 0.9749, 0.9543,\n",
      "        0.9654, 1.0036, 0.5330, 0.9355, 1.0091, 0.9156, 0.9495, 0.9479, 0.9375,\n",
      "        1.0115, 0.8691, 0.9082, 0.9674, 0.5240, 0.9454, 0.9314, 0.9685, 1.0078,\n",
      "        0.9736, 0.9812, 0.9921, 0.9372, 0.9798, 0.9541, 0.9280, 1.0170, 1.0109,\n",
      "        1.0155, 0.9896, 0.9223, 0.9387, 0.9520, 0.9600, 0.9065, 0.9002, 1.0119,\n",
      "        0.9207, 0.6654, 0.9034, 0.8686, 1.0040, 0.9420, 0.6352, 1.0383, 1.0030,\n",
      "        0.9751, 1.0054, 0.9720, 0.8924, 1.0244, 0.8549, 0.6884, 1.0259, 0.8896,\n",
      "        0.8655, 0.9604, 0.9462, 1.0118, 0.9887, 1.0270, 0.9212, 0.5036, 0.9239,\n",
      "        0.9525, 0.5487, 0.9234, 0.8468, 0.9412, 0.9195, 0.9978, 0.9039, 1.0132,\n",
      "        0.8649, 0.8634, 1.0373, 0.9632, 0.8643, 0.8053, 0.8553, 0.9282, 0.9895,\n",
      "        0.9845, 0.9282, 1.0061, 0.9784, 0.8048, 0.9694, 0.9862, 0.9485, 0.7485,\n",
      "        0.8998, 0.8824, 0.9730, 0.8973, 0.9850, 1.0194, 0.9851, 0.9382, 0.9495,\n",
      "        1.0178, 0.8283, 0.7034, 0.9299, 0.9114, 0.9404, 0.3123, 0.9734, 0.8809,\n",
      "        0.9638, 0.9532, 0.9335, 1.0294, 0.9856, 0.7874, 1.0428, 0.9160, 0.9379,\n",
      "        0.8645, 1.0035, 0.3188, 0.9947, 0.9320, 1.0182, 0.9271, 0.9392, 0.9952,\n",
      "        0.9663, 1.0906, 0.8087, 0.9176, 0.5875, 0.6196, 0.9788, 1.0025, 0.8913,\n",
      "        1.0460, 0.9528, 1.0008, 0.9377, 0.9846, 0.9472, 0.8246, 0.9669, 0.9796,\n",
      "        0.9673, 0.9996, 0.8727, 0.9388, 1.0101, 0.9653, 1.0194, 0.9267, 0.9475,\n",
      "        1.0497, 0.8756, 0.9595, 0.9755, 0.8934, 0.8506, 1.0381, 0.9226, 0.8830,\n",
      "        0.8069, 0.9708, 1.0265, 0.9560, 1.0542, 1.0082, 0.9440, 1.0219, 0.2311,\n",
      "        0.9968, 1.0137, 0.9566, 0.9089, 0.9145, 0.9271, 0.7979, 0.9993, 1.0164,\n",
      "        1.0025, 1.0131, 0.9014, 0.8750, 0.9209, 1.0402, 0.9325, 0.9954, 0.9632,\n",
      "        1.0002, 0.9340, 0.9082, 0.9120, 0.9181, 0.5339, 0.9546, 0.9217, 0.9401,\n",
      "        0.9152, 1.0033, 1.0335, 0.9293, 0.8252, 0.9830, 0.9219, 0.9965, 1.0004,\n",
      "        0.9608, 1.0047, 0.8758, 0.9887, 0.9977, 0.8867, 1.0055, 0.8432, 0.9680,\n",
      "        0.3345, 0.6632, 0.8753, 0.9532, 0.8407, 0.9724, 0.9846, 0.9296, 0.9618,\n",
      "        0.9561, 0.9908, 0.9130, 0.9438, 0.9983, 0.9067, 0.8133, 0.9210, 0.9068,\n",
      "        0.8879, 0.9278, 0.8713, 0.9882, 0.9694, 1.0173, 0.9140, 0.7836, 0.8972,\n",
      "        0.9013, 0.8586, 0.9572, 0.3050, 0.9389, 0.8438, 1.0235, 0.9100, 0.8846,\n",
      "        0.7097, 0.8667, 0.8627, 0.9841, 0.9952, 0.9769, 0.7431, 0.9481, 0.8583,\n",
      "        0.9524, 0.9591, 0.7028, 0.9554, 1.0403, 0.9646])), ('bert.embeddings.LayerNorm.bias', tensor([ 1.5071e-01,  2.1815e-02,  9.7619e-02,  5.3961e-02,  4.7023e-02,\n",
      "         5.1321e-03,  8.4385e-02, -5.3549e-02, -5.4150e-03,  7.5155e-02,\n",
      "         9.0195e-02,  4.1364e-02,  8.7828e-02,  6.0487e-02,  8.0172e-03,\n",
      "         1.2408e-01,  3.3006e-01,  8.9289e-02,  1.0427e-01, -1.8356e-02,\n",
      "         5.6539e-02,  4.2258e-02,  4.6504e-02, -7.8949e-03, -2.5826e-02,\n",
      "         1.8014e-01,  3.2066e-03, -3.1136e-04,  3.2065e-02,  1.1139e-01,\n",
      "         1.5321e-01,  1.0780e-01,  5.8418e-02,  4.4057e-02,  6.6491e-02,\n",
      "         9.4266e-02,  9.3011e-02,  1.2339e-01, -1.4274e-03,  1.0557e-02,\n",
      "        -1.4732e-02,  9.8584e-02,  4.6812e-02,  3.3242e-02, -6.8395e-02,\n",
      "         2.6780e-01,  4.7355e-02, -4.7907e-02,  1.0653e-01,  5.3204e-03,\n",
      "         2.6875e-02,  4.8117e-03, -9.7503e-02,  2.0102e-02,  8.0210e-03,\n",
      "         1.1538e-01,  3.6723e-02,  1.0385e-01,  3.4165e-02,  6.3876e-02,\n",
      "         1.5677e-02,  1.1803e-01, -1.2738e-03,  2.5622e-02,  1.0927e-02,\n",
      "         1.2816e-01, -4.8661e-02,  5.5879e-02, -1.5149e-01,  3.4938e-02,\n",
      "        -6.3371e-02,  6.9388e-02, -2.6143e-02, -1.0625e-02,  1.9867e-01,\n",
      "        -4.7584e-02,  1.6772e-02,  3.2041e-02,  1.0032e-01,  1.1091e-01,\n",
      "        -2.0026e-01, -4.2243e-01, -3.8989e-02,  1.7834e-02,  4.7161e-02,\n",
      "        -8.9828e-02,  2.8246e-02, -1.0016e-01,  1.6391e-01, -1.7839e-01,\n",
      "         1.4088e-02,  9.0101e-02,  3.3690e-02, -9.0554e-02, -5.7760e-02,\n",
      "        -1.3617e-01,  7.4066e-03,  1.9971e-02,  5.6711e-02,  8.3541e-03,\n",
      "         5.3232e-02, -2.3369e-01, -2.2902e-02,  3.5034e-02,  8.4562e-02,\n",
      "        -3.6914e-01,  2.2484e-02, -1.0119e-01,  7.2892e-02, -1.2001e-01,\n",
      "         4.8882e-03, -3.8627e-02,  5.2461e-02, -1.1046e-01,  1.5381e-02,\n",
      "         8.8666e-02, -9.8495e-03, -3.3415e-02,  2.5845e-02, -7.7632e-02,\n",
      "        -1.9863e-02,  5.1835e-02, -4.6567e-03, -3.9701e-02, -1.2943e-02,\n",
      "         4.2114e-02, -2.5572e-02, -1.3489e-01,  6.8808e-02, -6.8205e-02,\n",
      "        -2.1130e-01, -1.0234e-02,  3.9939e-02,  4.8393e-02,  5.9458e-03,\n",
      "        -3.4811e-02,  1.0794e-01, -1.9499e-02,  1.0050e-01, -4.2221e-02,\n",
      "        -1.0036e-02, -5.8697e-02,  5.1578e-02, -5.0749e-02,  4.5790e-02,\n",
      "         6.5893e-02,  1.2520e-02,  2.9895e-02,  8.4379e-02, -4.3203e-02,\n",
      "         6.6759e-02, -4.3004e-02,  2.8667e-02,  4.1738e-02,  8.4699e-02,\n",
      "        -9.1400e-02,  1.4136e-01, -1.1740e-01,  5.7158e-02, -2.5982e-01,\n",
      "         5.3269e-02,  2.1647e-02,  8.9721e-02, -8.7351e-03, -1.6299e-03,\n",
      "        -1.4335e-01,  5.5077e-02, -1.0035e-01,  1.6920e-01, -6.6230e-02,\n",
      "        -7.4492e-02, -2.9380e-02,  3.8053e-02, -8.9141e-02,  9.7600e-02,\n",
      "         5.4220e-02,  5.4592e-02,  6.1955e-02,  1.2334e-02, -7.2175e-02,\n",
      "         9.1075e-02, -1.4774e-01,  6.4650e-02,  1.2986e-02,  1.3900e-01,\n",
      "         9.2114e-02,  1.1866e-02, -5.3845e-01, -6.6052e-03, -1.2980e-01,\n",
      "         9.1216e-02, -3.2742e-02,  4.5266e-02, -7.0626e-03,  1.6291e-01,\n",
      "        -2.7113e-02, -1.3377e-03, -1.5242e-02, -3.4666e-02, -9.4011e-02,\n",
      "         1.6418e-01, -2.4928e-02,  1.7113e-01, -6.6011e-02,  7.6588e-02,\n",
      "        -4.4833e-02, -6.7339e-02, -9.7252e-02,  7.4424e-02, -6.9104e-02,\n",
      "         2.2160e-03, -5.0540e-03,  6.7066e-02, -6.0394e-02, -9.0711e-03,\n",
      "        -8.0344e-02,  8.8976e-03, -8.4834e-02,  1.5097e-01, -3.9238e-02,\n",
      "         1.1786e-01, -7.8122e-02,  1.5484e-01, -6.5550e-02,  1.6213e-01,\n",
      "        -3.9433e-02,  2.0863e-02, -1.7731e-03, -3.8526e-02, -2.0422e-02,\n",
      "         5.6978e-02, -3.2720e-02,  1.6313e-01, -1.0967e-01,  1.0414e-01,\n",
      "        -2.4209e-02,  7.4095e-02,  8.4043e-03,  7.2131e-02,  5.3192e-03,\n",
      "         1.1135e-01, -6.0503e-02,  5.0093e-04, -5.0609e-02,  8.7454e-02,\n",
      "        -8.2325e-02,  1.2436e-01, -8.1536e-02,  1.1006e-01, -3.1538e-02,\n",
      "        -1.3693e-02, -3.3337e-03,  5.9029e-02,  1.1372e-01, -1.7582e-02,\n",
      "        -2.9196e-03,  8.3875e-02,  3.1100e-02,  2.1287e-01, -6.8272e-04,\n",
      "         4.5090e-03, -2.2954e-02,  6.7049e-02,  3.1509e-02,  2.1464e-02,\n",
      "        -3.2279e-02,  3.6498e-02, -7.4930e-02,  5.0294e-02, -9.3838e-02,\n",
      "         4.8041e-02, -9.2465e-02,  8.0296e-02, -6.2057e-02,  1.3074e-01,\n",
      "        -8.3195e-02,  8.6196e-02, -3.4308e-02, -2.4177e-02, -1.4476e-01,\n",
      "         2.0932e-02, -4.4751e-03,  6.4529e-02,  1.4240e-01,  8.2654e-02,\n",
      "        -4.3840e-02,  6.3305e-03, -1.9221e-01,  8.6327e-02, -5.9892e-02,\n",
      "         7.4827e-02, -8.6376e-02,  1.2882e-01,  2.1877e-03,  1.7558e-01,\n",
      "         1.0384e-03,  5.2535e-02,  1.0363e-02,  8.6800e-02, -2.4569e-02,\n",
      "         5.5980e-02, -3.8143e-02, -2.6388e-02, -3.6439e-02,  1.2805e-01,\n",
      "        -1.1011e-02,  4.8903e-02, -6.4550e-02,  1.2570e-01, -7.5689e-02,\n",
      "         2.2230e-02, -4.5923e-02,  5.3771e-02, -6.1537e-02,  7.5809e-02,\n",
      "         1.5899e-03,  7.3000e-02, -5.8078e-02,  4.8949e-03, -5.8930e-02,\n",
      "         5.1090e-02,  9.6407e-03,  9.7765e-02, -5.7193e-02,  4.2690e-02,\n",
      "        -8.4735e-02,  2.1159e-02,  2.5385e-02,  3.0281e-02, -8.1405e-03,\n",
      "         6.7716e-02, -1.5566e-02,  7.4765e-02, -1.0709e-01,  4.3833e-02,\n",
      "        -8.1000e-02,  5.7513e-03,  6.4300e-02,  3.5466e-02,  4.8706e-02,\n",
      "        -5.1595e-03, -7.5425e-02, -6.8725e-03, -4.8564e-02,  8.5218e-02,\n",
      "         1.2340e-02, -3.6100e-02, -2.3461e-02,  2.3722e-01,  2.3763e-02,\n",
      "         1.3063e-01, -3.4395e-02,  1.5948e-01, -6.8108e-02,  9.3199e-02,\n",
      "        -1.3200e-01,  8.1992e-02, -9.1941e-02,  1.5971e-01, -1.0927e-01,\n",
      "         3.7609e-03, -1.2030e-02,  1.1633e-01,  1.3936e-03, -6.4780e-03,\n",
      "         1.4355e-02,  3.2097e-02, -1.2286e-01,  8.2700e-03, -1.8610e-01,\n",
      "         1.0061e-01, -8.3970e-02,  1.0398e-01, -6.3737e-03,  5.6982e-02,\n",
      "        -4.1379e-02, -4.1181e-02, -9.2600e-02,  1.1913e-01, -9.6142e-02,\n",
      "        -7.6481e-02,  2.4111e-02,  6.1950e-02, -4.7645e-03])), ('bert.encoder.layer.0.attention.self.query.weight', tensor([[ 0.0003, -0.0236,  0.0111,  ...,  0.0242, -0.0693,  0.0211],\n",
      "        [ 0.0307,  0.0021,  0.0190,  ..., -0.0284,  0.0479,  0.0200],\n",
      "        [-0.0312,  0.0243,  0.0124,  ..., -0.0645,  0.0256, -0.0210],\n",
      "        ...,\n",
      "        [ 0.0124,  0.0516,  0.0099,  ...,  0.0230,  0.0295,  0.0379],\n",
      "        [ 0.0163,  0.0487,  0.0204,  ...,  0.0121,  0.0317,  0.0138],\n",
      "        [-0.0169,  0.0121,  0.0098,  ..., -0.0374, -0.0096,  0.0162]])), ('bert.encoder.layer.0.attention.self.query.bias', tensor([ 2.3620e-02, -1.5525e-01, -5.3429e-02,  1.2674e-01,  8.1202e-02,\n",
      "         2.1125e-01,  1.8301e-01,  6.0979e-02, -2.3159e-01, -4.6049e-02,\n",
      "         1.3815e-01,  4.3272e-02, -6.3043e-02,  4.2381e-02,  2.0968e-02,\n",
      "        -2.5311e-01,  7.3410e-02, -5.7718e-02, -9.5609e-02,  2.0894e-01,\n",
      "         4.2952e-03, -1.2976e-01,  9.8480e-02,  6.1343e-02, -2.5908e-01,\n",
      "        -1.4918e-01, -1.3187e-01,  5.5754e-02,  1.1249e-03,  1.1269e-01,\n",
      "        -1.2651e-01, -4.9769e-02, -4.1656e-02, -3.3753e-01,  1.7377e-01,\n",
      "         2.2670e-01,  1.0046e-02, -1.8290e-01, -3.0728e-01,  3.3144e-01,\n",
      "        -2.7952e-01, -2.7836e-02, -4.3237e-03,  3.8500e-02, -2.3833e-01,\n",
      "        -4.2006e-02, -1.3560e-01, -3.4041e-01, -2.5609e-01, -5.4483e-02,\n",
      "         2.2436e-01, -2.4414e-01, -1.8224e-03,  2.1403e-01, -8.2714e-02,\n",
      "         1.2203e-01, -8.7205e-02, -3.2825e-01,  1.6346e-01,  2.9505e-02,\n",
      "         1.2320e-01,  2.5817e-01, -1.3257e-01,  1.7081e-01, -3.4977e-02,\n",
      "        -2.2495e-02, -5.8306e-02,  7.7918e-02, -2.5305e-02,  2.6365e-01,\n",
      "        -5.6616e-02, -6.0764e-02, -1.7227e-01, -1.2937e-02,  4.6409e-02,\n",
      "         2.8429e-01, -8.0058e-03, -2.5908e-01, -1.0370e-01,  2.9799e-02,\n",
      "        -1.6573e-01, -7.6832e-03,  2.2082e-01, -2.3949e-02,  1.1406e-01,\n",
      "         1.4385e-01, -1.9157e-01, -2.8795e-01,  2.0234e-01,  2.1592e-02,\n",
      "         1.4291e-01, -1.1486e-01, -7.0689e-02, -1.3056e-01, -1.8910e-01,\n",
      "         4.8932e-02, -5.2677e-03,  2.7324e-01, -1.1165e-01,  5.7431e-02,\n",
      "         1.3318e-01,  8.6056e-03, -7.4003e-02,  2.7791e-01, -3.2511e-02,\n",
      "         1.7267e-01, -2.5431e-02,  1.7158e-02,  2.4936e-01, -4.1290e-01,\n",
      "        -4.2882e-02,  1.7190e-02, -7.1762e-02,  3.7178e-01, -5.9547e-02,\n",
      "        -1.5261e-01, -3.5010e-02,  4.6147e-01,  2.3205e-01, -1.2376e-04,\n",
      "         7.8471e-02, -1.8786e-01, -1.1043e-01, -6.0357e-02, -8.5597e-02,\n",
      "         2.5974e-01,  2.3686e-01,  2.9295e-01,  1.5658e-01, -1.2193e-01,\n",
      "         1.8616e-01,  8.5181e-02,  9.0302e-02, -8.1765e-02,  5.1702e-02,\n",
      "         5.9715e-02, -1.8438e-01, -5.4261e-02,  1.8693e-01,  1.4846e-02,\n",
      "        -2.4322e-01,  5.1479e-02, -5.7273e-02, -2.2403e-01, -4.3070e-02,\n",
      "        -8.4615e-02, -8.4780e-02, -8.7599e-02, -1.2491e-01,  2.4854e-02,\n",
      "        -1.0791e-01, -3.9056e-02,  1.8358e-03, -1.1030e-02, -1.5139e-02,\n",
      "         2.2363e-02,  1.6051e-01,  6.9985e-02, -6.0279e-02,  7.0036e-03,\n",
      "         4.0773e-02, -3.8258e-03,  6.3328e-02,  1.1973e-01,  1.2712e-01,\n",
      "        -1.2180e-01,  2.1323e-03,  1.3002e-01,  1.0495e-01, -2.1120e-01,\n",
      "        -7.7421e-02,  1.2801e-01,  1.1392e-01,  1.2055e-01, -2.5425e-02,\n",
      "         7.4864e-02, -6.9621e-02,  1.4922e-01,  1.0006e-01, -1.0407e-01,\n",
      "         4.6560e-02,  1.1627e-01, -7.3190e-03, -6.2593e-02,  1.1403e-01,\n",
      "         8.2209e-02,  1.1641e-01, -5.7482e-02,  2.9590e-02,  6.0960e-02,\n",
      "        -3.1191e-02,  1.0931e-01, -5.1425e-02, -1.6185e-01,  1.5704e-01,\n",
      "         2.3940e-02, -1.7903e-01,  1.5685e-01,  9.0849e-03,  3.8471e-02,\n",
      "         1.9851e-01,  1.6279e-01, -4.9635e-02, -9.2776e-02,  1.3500e-01,\n",
      "         1.6510e-01,  4.0754e-02,  2.1377e-01,  9.1840e-02,  4.5232e-02,\n",
      "         5.9612e-02,  2.0657e-01,  2.7065e-01, -1.2473e-01, -1.0539e-01,\n",
      "         6.6126e-02,  1.1216e-01, -4.6447e-03, -1.6982e-02, -2.9458e-01,\n",
      "         1.9039e-01,  1.5850e-01,  1.1086e-01,  2.5550e-02,  8.9589e-02,\n",
      "        -6.2992e-02,  1.5816e-01,  1.5333e-01, -2.6879e-01,  1.2250e-01,\n",
      "        -8.5874e-02,  9.4368e-02, -3.6772e-02,  1.1348e-01, -1.5749e-01,\n",
      "         6.1028e-02,  3.6373e-02, -7.4403e-02, -6.4354e-02,  3.5878e-02,\n",
      "        -1.3855e-01, -8.6528e-02,  2.3776e-02, -1.2885e-01, -3.6833e-02,\n",
      "         8.8628e-04, -1.9202e-01, -3.2252e-02,  9.1523e-02,  7.1677e-03,\n",
      "         8.0071e-02, -1.9971e-01, -4.7281e-02,  1.2996e-01,  4.8292e-02,\n",
      "        -1.9192e-01,  7.3374e-02, -6.4456e-02,  3.8469e-02, -9.1974e-02,\n",
      "         5.1395e-02, -1.3957e-01,  6.2021e-02, -7.7661e-02,  9.1418e-02,\n",
      "         1.3030e-01,  7.8498e-02,  1.0720e-02, -4.9579e-02,  1.1343e-01,\n",
      "         4.0921e-02, -1.1061e-03,  1.6081e-01, -6.5880e-02,  1.5537e-01,\n",
      "         8.5629e-02,  3.0967e-01,  3.9176e-02, -9.3925e-02, -5.4144e-02,\n",
      "         2.9305e-02, -1.0474e-02,  1.0199e-01,  1.8002e-02, -5.4172e-04,\n",
      "        -3.1858e-02,  6.5745e-03,  8.8956e-02,  1.5401e-02, -1.4799e-01,\n",
      "         8.0969e-02,  4.2929e-02,  2.0244e-01,  1.7482e-01,  7.8158e-02,\n",
      "        -4.1144e-02,  2.8996e-02, -1.2349e-01,  2.5645e-01, -1.4794e-01,\n",
      "         4.7550e-02,  7.5933e-02,  9.6895e-02, -7.0435e-02, -4.2505e-02,\n",
      "         1.0190e-01,  8.5203e-03,  1.7007e-01, -2.5671e-02, -2.3187e-02,\n",
      "        -1.8478e-01,  6.9700e-03,  1.3629e-01,  1.7713e-01, -2.3477e-02,\n",
      "        -1.3237e-01, -1.0527e-01, -1.6372e-01, -1.9774e-01,  1.0490e-02,\n",
      "         4.8069e-02,  2.1816e-02,  5.0299e-02,  1.5212e-01, -7.4688e-02,\n",
      "        -1.6044e-01,  9.3915e-02,  1.1006e-01, -1.9048e-01,  1.2395e-01,\n",
      "         9.4723e-02, -1.5181e-02, -1.9228e-01, -1.2093e-01, -6.5761e-02,\n",
      "         1.2813e-02, -1.6058e-01, -3.7046e-02,  7.0263e-02, -8.4210e-02,\n",
      "         1.5419e-01, -1.6942e-02, -8.1928e-02, -7.8066e-02,  4.7082e-02,\n",
      "        -1.3869e-01,  6.8246e-03,  1.1373e-02,  2.6787e-02, -9.3168e-02,\n",
      "         4.9860e-03,  9.1775e-02, -1.1997e-01,  1.1830e-01, -3.0345e-01,\n",
      "         3.1524e-03, -1.0596e-01, -1.2569e-01, -1.5769e-02, -1.0164e-01,\n",
      "        -5.8032e-02, -1.0898e-01, -1.2405e-01,  1.0179e-01,  7.8311e-02,\n",
      "         1.9900e-02,  3.2240e-01, -9.8943e-03, -7.2383e-02,  2.7650e-02,\n",
      "         5.2253e-03, -2.1519e-02, -2.3184e-01,  4.9603e-02, -6.8642e-02,\n",
      "        -1.3302e-01, -1.2872e-01, -1.3058e-01, -3.4306e-01, -1.7227e-01,\n",
      "        -5.0384e-02,  1.8993e-01, -1.0364e-01, -1.4925e-01])), ('bert.encoder.layer.0.attention.self.key.weight', tensor([[ 0.0082, -0.0064, -0.0309,  ...,  0.0391, -0.1047, -0.0088],\n",
      "        [-0.0524,  0.0303,  0.0022,  ..., -0.0233,  0.0848, -0.0157],\n",
      "        [-0.0060, -0.0182,  0.0124,  ...,  0.0132,  0.1058,  0.0147],\n",
      "        ...,\n",
      "        [-0.0213, -0.0260, -0.0137,  ...,  0.0026, -0.0181, -0.0148],\n",
      "        [-0.0098, -0.0517, -0.0351,  ..., -0.0195,  0.0330,  0.0319],\n",
      "        [ 0.0020, -0.0064, -0.0253,  ..., -0.0030, -0.0057, -0.0148]])), ('bert.encoder.layer.0.attention.self.key.bias', tensor([ 1.6199e-04,  1.8824e-04,  9.3487e-04,  1.0239e-03,  1.6878e-06,\n",
      "         4.4743e-04,  4.3444e-05,  4.2034e-04,  7.1963e-04, -6.6038e-04,\n",
      "        -2.5623e-04,  7.9340e-04, -3.3589e-04,  6.1721e-04, -7.8936e-04,\n",
      "         1.2690e-03, -1.0718e-03,  1.9255e-04, -4.7852e-04,  7.7614e-04,\n",
      "         3.1672e-04,  3.1954e-04,  1.4974e-03, -8.6862e-04, -5.6124e-04,\n",
      "        -1.7103e-04, -8.1485e-04,  1.3970e-04,  6.4138e-05,  5.0645e-04,\n",
      "         1.3302e-03, -1.6081e-04, -3.1426e-04, -2.9033e-04, -3.1384e-04,\n",
      "        -5.7229e-05, -8.5344e-04, -2.9986e-05, -4.2159e-04,  2.4090e-04,\n",
      "         7.9997e-05,  9.9422e-04,  8.5772e-04,  4.1746e-04, -1.1944e-03,\n",
      "        -5.7509e-04, -2.6333e-04, -6.7377e-05,  1.6523e-04, -3.2848e-04,\n",
      "         8.8484e-04,  7.3728e-05,  1.3362e-04,  1.5503e-04, -9.1274e-04,\n",
      "         1.3447e-05, -6.1237e-04,  2.0652e-04,  6.4079e-04,  6.5944e-04,\n",
      "        -7.8630e-04, -6.8044e-04,  2.9442e-04, -1.2399e-03,  1.1859e-03,\n",
      "         8.5991e-04,  1.4250e-03, -3.1198e-04, -9.7403e-04, -1.9364e-04,\n",
      "        -1.6736e-03,  8.6563e-04, -5.2967e-04, -1.5894e-03,  1.4398e-04,\n",
      "         1.1439e-03, -1.2306e-04,  6.4075e-05, -6.2192e-04,  8.5670e-04,\n",
      "         5.0356e-03, -3.0252e-04, -3.0492e-03,  2.2309e-03,  6.6084e-04,\n",
      "        -2.3016e-03, -8.5359e-05,  1.8792e-03,  2.0123e-03,  2.8876e-03,\n",
      "        -3.0929e-03,  2.8813e-03,  1.8009e-03,  6.0013e-04,  2.4028e-03,\n",
      "        -1.8041e-03,  1.5933e-03,  2.3403e-03,  2.5116e-03,  3.7791e-04,\n",
      "        -9.8940e-04,  1.3658e-03, -1.4609e-03, -5.0830e-04, -1.0371e-04,\n",
      "        -4.1357e-04, -5.2363e-04, -2.8600e-04, -1.9576e-03, -2.4093e-03,\n",
      "        -1.5834e-03, -2.9845e-03,  1.3017e-03,  1.6053e-03, -9.1893e-05,\n",
      "        -2.7306e-04,  5.3933e-05,  1.4771e-03,  1.9647e-04,  6.4139e-04,\n",
      "         1.7790e-03, -9.9915e-04,  6.1584e-04, -2.6273e-04, -8.4042e-04,\n",
      "        -1.8159e-03, -1.6652e-04,  9.8667e-04,  5.7534e-05, -6.4968e-05,\n",
      "         6.2342e-04,  2.0808e-04,  5.4986e-05, -1.4700e-04, -1.0626e-04,\n",
      "         7.3646e-04,  1.5299e-05, -1.5993e-05,  1.7717e-04, -6.0770e-05,\n",
      "        -2.1129e-04, -7.7357e-04,  4.5798e-04, -4.8818e-05,  6.8125e-04,\n",
      "         1.9542e-04, -2.2153e-04,  1.8839e-04, -8.7141e-04,  4.2201e-04,\n",
      "         1.7274e-04,  5.2722e-04, -1.1596e-04, -4.0887e-05, -5.3098e-04,\n",
      "         6.9888e-04,  5.2525e-04, -5.1900e-06,  8.9502e-04, -2.1388e-04,\n",
      "         1.4014e-03, -5.3859e-04,  9.1528e-04, -2.0894e-04,  2.6018e-04,\n",
      "         7.8497e-04, -5.7860e-04, -2.5110e-05, -6.0838e-04, -2.1240e-04,\n",
      "        -1.4836e-03,  4.6849e-04,  1.9448e-03,  2.1106e-03, -8.9906e-04,\n",
      "        -2.6516e-04, -2.2247e-04, -2.9912e-04,  7.5488e-04, -1.1355e-03,\n",
      "        -1.4279e-04, -3.4232e-04,  1.9181e-04, -1.2978e-03,  2.5310e-03,\n",
      "         1.3432e-04,  1.5658e-03, -6.2047e-04,  1.5655e-03, -1.3788e-03,\n",
      "        -5.2302e-04, -6.4060e-04, -4.7627e-04, -7.0256e-06,  2.8463e-04,\n",
      "         4.8763e-05,  6.7432e-05, -3.8360e-04, -6.5883e-04, -1.8356e-03,\n",
      "        -2.5608e-04, -3.4371e-04, -1.8142e-04,  7.8081e-04,  3.6185e-04,\n",
      "         2.2380e-04, -1.4181e-03, -1.5231e-03,  4.6163e-04, -2.6795e-04,\n",
      "        -5.7578e-04, -1.2473e-04, -5.2848e-04,  1.0171e-03,  2.5633e-04,\n",
      "        -6.1853e-04, -9.5346e-04, -3.4402e-04, -4.2433e-05,  1.3243e-04,\n",
      "        -1.7708e-04, -1.7968e-04, -3.9105e-04,  1.2762e-03,  1.3957e-03,\n",
      "         6.4852e-05, -4.1978e-04,  1.4087e-03, -4.7869e-04, -2.6580e-03,\n",
      "         1.1165e-04,  3.0751e-04, -1.9052e-04,  1.1689e-03,  1.0679e-04,\n",
      "         1.1546e-03,  1.6196e-03, -9.1346e-04, -1.2254e-03,  7.8472e-04,\n",
      "        -5.9109e-04,  3.7684e-04, -6.0715e-04, -1.4235e-03, -2.4214e-03,\n",
      "         7.5772e-04, -2.7465e-04,  1.4974e-03, -5.5386e-07, -2.4899e-04,\n",
      "         7.1402e-04, -1.4504e-03, -7.2615e-04, -2.0928e-03,  1.2822e-03,\n",
      "        -8.3482e-04, -8.0471e-04, -1.7234e-04, -8.9103e-04, -1.2238e-03,\n",
      "         4.9702e-04,  4.0713e-04, -5.4296e-05, -8.0873e-04, -7.8976e-04,\n",
      "         1.0948e-03, -1.2414e-03, -2.4735e-04, -1.4177e-03,  6.2055e-04,\n",
      "        -6.2840e-04, -9.2247e-04,  1.1614e-03, -9.6326e-04, -1.4252e-03,\n",
      "         2.9966e-03,  4.9782e-05, -1.2303e-04,  5.6790e-04,  2.8335e-03,\n",
      "        -8.9433e-04, -7.1872e-04,  4.5876e-04, -4.3466e-04, -3.0037e-05,\n",
      "        -4.8844e-04, -1.1833e-03,  2.1152e-03,  2.0537e-04, -2.0459e-03,\n",
      "        -4.2862e-04,  1.1481e-03, -1.1854e-03, -3.8009e-04, -7.4474e-05,\n",
      "        -4.7959e-06,  4.8614e-06,  3.0248e-04, -2.8398e-03,  3.6129e-04,\n",
      "        -1.7095e-03, -2.8654e-04, -1.0206e-03,  1.9972e-03,  1.5457e-03,\n",
      "         1.7351e-03,  1.4509e-03, -4.4302e-04, -1.0680e-03, -2.0952e-03,\n",
      "        -1.5454e-03, -5.2598e-05,  8.0612e-04, -2.2805e-03,  5.3438e-05,\n",
      "        -1.7226e-04, -7.6566e-04,  5.8862e-04,  1.8621e-03,  6.7661e-04,\n",
      "        -1.2081e-03,  1.8682e-04,  5.5652e-04,  1.6565e-04, -1.9303e-03,\n",
      "        -8.0223e-04,  2.9746e-03,  6.7975e-04,  1.9472e-04,  9.7929e-04,\n",
      "         7.3745e-04,  5.5607e-04,  3.5480e-04, -9.8586e-04, -5.4693e-05,\n",
      "        -5.2106e-04,  7.6888e-04, -8.1398e-04,  1.0055e-03,  3.4009e-05,\n",
      "        -6.2494e-04,  1.7336e-03,  4.3955e-04,  1.7111e-03, -1.2084e-03,\n",
      "        -2.6249e-04,  1.8125e-03, -9.0365e-04,  1.0436e-03, -1.9660e-03,\n",
      "        -2.2385e-03, -1.2632e-03,  1.4683e-03,  9.3326e-05, -2.1668e-03,\n",
      "        -2.6646e-04, -1.0954e-03,  1.0941e-03, -2.8817e-03,  6.3310e-04,\n",
      "        -6.0902e-04, -2.4808e-04,  8.3012e-04,  9.9908e-04,  5.1029e-04,\n",
      "         1.0262e-04, -4.1301e-04, -1.1820e-03,  4.8646e-04, -4.5777e-04,\n",
      "        -2.0766e-03, -1.1417e-03,  4.7656e-04,  5.0410e-04,  5.2404e-04,\n",
      "        -1.1610e-03,  8.1373e-04, -1.8509e-04,  6.2978e-04,  1.1653e-03,\n",
      "        -7.4761e-04,  1.5442e-03, -5.3054e-04, -1.9397e-04])), ('bert.encoder.layer.0.attention.self.value.weight', tensor([[-0.0019,  0.0066, -0.0016,  ..., -0.0925, -0.0102,  0.0056],\n",
      "        [ 0.0264,  0.0505,  0.0142,  ...,  0.0404,  0.0012,  0.0571],\n",
      "        [-0.0124,  0.0031, -0.0439,  ..., -0.0479,  0.0194, -0.0230],\n",
      "        ...,\n",
      "        [ 0.0031, -0.0029, -0.0046,  ...,  0.0676, -0.0251, -0.0123],\n",
      "        [ 0.0197, -0.0049, -0.0018,  ..., -0.0074, -0.0509,  0.0128],\n",
      "        [ 0.0211, -0.0106, -0.0023,  ..., -0.0027, -0.0052, -0.0563]])), ('bert.encoder.layer.0.attention.self.value.bias', tensor([ 1.6461e-02, -3.5035e-03,  7.9467e-03, -3.2245e-02,  2.0775e-02,\n",
      "        -3.5876e-03,  3.3284e-03, -9.5623e-03,  9.9883e-03,  2.4097e-02,\n",
      "         2.1709e-04,  6.2799e-03, -1.2150e-02, -1.4086e-02,  9.3090e-03,\n",
      "        -7.5116e-03,  1.1572e-02, -1.3310e-02,  6.2700e-04,  1.4048e-02,\n",
      "        -2.5560e-05,  2.1684e-02, -9.7201e-03,  8.7002e-03, -9.0358e-04,\n",
      "        -2.2977e-02,  1.2679e-03,  1.7392e-02, -1.6981e-02, -1.9165e-02,\n",
      "        -6.4082e-03, -1.2693e-02,  1.0949e-02, -3.2389e-03,  1.2596e-03,\n",
      "         7.2356e-02, -5.7573e-02,  5.2142e-02, -1.1338e-02,  1.3962e-02,\n",
      "         1.7437e-02,  2.5875e-02,  1.5908e-02,  9.3294e-03,  4.6484e-02,\n",
      "         4.6339e-02,  5.1035e-03, -3.7711e-02, -1.2524e-02,  4.1979e-02,\n",
      "        -3.2518e-02,  6.0719e-02,  1.5079e-05, -1.7938e-02,  1.0465e-02,\n",
      "        -8.4398e-03,  1.9805e-02, -1.8239e-02,  2.5649e-02,  4.4975e-02,\n",
      "         2.8227e-03,  2.2535e-03,  8.7780e-03, -2.0714e-02,  5.5906e-03,\n",
      "         2.4740e-03,  1.7992e-02, -7.0736e-03,  6.7897e-03,  1.2215e-02,\n",
      "         2.9907e-02, -1.0351e-03, -1.5210e-02, -1.3414e-02,  8.3972e-04,\n",
      "        -3.6000e-03,  1.3899e-02,  9.5008e-03, -1.8101e-02,  6.1701e-03,\n",
      "        -6.2284e-03, -1.4558e-02,  1.3805e-02, -1.1274e-02,  2.2032e-02,\n",
      "         1.1450e-02,  4.8564e-03,  1.0724e-02,  2.2101e-02,  1.8357e-02,\n",
      "        -1.0386e-02,  5.3112e-03,  5.8436e-03, -9.5265e-03, -9.1274e-03,\n",
      "        -1.6865e-03,  3.4206e-03,  1.9003e-02,  1.5917e-02, -9.6881e-03,\n",
      "         1.0381e-02, -3.5427e-03, -2.4239e-02, -1.6910e-02,  8.2253e-03,\n",
      "        -1.0981e-03, -8.7652e-03,  6.8700e-03,  9.1927e-03, -1.3668e-02,\n",
      "         2.7810e-02, -2.1149e-03, -9.7156e-03, -1.7576e-02,  1.8936e-02,\n",
      "         3.2679e-03, -1.0209e-02, -7.4045e-03, -2.8913e-02,  1.6916e-02,\n",
      "         3.9399e-03, -5.0653e-03, -2.4641e-02,  3.3664e-03, -1.5206e-02,\n",
      "        -2.3482e-02, -7.6694e-03, -1.2033e-02,  1.7710e-02, -2.1133e-03,\n",
      "         1.3582e-03,  7.9886e-03,  1.0152e-02,  3.3370e-02,  7.2496e-03,\n",
      "         5.4449e-03, -1.8921e-02, -9.2236e-03, -4.9763e-03,  5.8911e-04,\n",
      "         5.1413e-03, -7.7753e-03, -2.3180e-02, -1.0208e-02, -8.0166e-03,\n",
      "        -5.3513e-04,  1.5700e-02,  4.3110e-02,  1.6858e-02,  7.5926e-03,\n",
      "         2.6544e-02, -1.2916e-02,  4.3793e-03,  3.6953e-03, -3.3028e-03,\n",
      "        -9.4747e-03, -1.0437e-02, -1.1409e-03,  1.1827e-02,  2.0195e-02,\n",
      "        -1.1420e-02,  7.0143e-03,  1.2315e-03, -2.4227e-02, -9.1541e-03,\n",
      "        -2.2560e-02,  7.5886e-03,  1.2444e-02,  1.7650e-02, -8.9364e-03,\n",
      "        -1.4753e-02, -1.0278e-02, -3.5213e-03, -1.2578e-02,  1.7509e-02,\n",
      "         1.6986e-02, -1.3051e-02, -2.5508e-02,  6.0325e-03, -1.3371e-02,\n",
      "        -6.8102e-03,  1.5928e-02,  8.0863e-03,  1.2164e-02, -4.4335e-05,\n",
      "        -6.8950e-03, -3.0654e-03, -2.6757e-03,  1.4191e-02,  9.1249e-03,\n",
      "        -8.9920e-03,  1.9013e-03,  1.0825e-04,  1.8631e-02,  2.2104e-02,\n",
      "        -1.2316e-02,  1.2982e-02,  1.3094e-02,  6.8038e-03, -1.4295e-02,\n",
      "         9.3963e-05, -1.2136e-02,  2.6143e-02, -2.3096e-02, -2.6743e-03,\n",
      "         1.2552e-02,  1.5494e-02,  4.4188e-03, -3.3671e-03, -3.4692e-02,\n",
      "        -1.6628e-02, -8.8771e-03, -2.6583e-02,  4.1851e-03, -4.5077e-03,\n",
      "        -2.8151e-02,  3.3115e-03,  1.2480e-02,  1.6879e-02,  2.7432e-02,\n",
      "        -3.6801e-03,  1.4227e-02, -1.2947e-03, -8.7803e-03, -2.8749e-02,\n",
      "        -8.2637e-03,  3.2966e-03,  1.5853e-02,  1.0980e-02, -2.4931e-02,\n",
      "        -5.7083e-03,  3.4974e-02,  3.1034e-02, -4.8796e-02, -3.3803e-02,\n",
      "        -8.2129e-04,  8.2171e-02,  2.4982e-02,  3.7230e-02, -4.7305e-03,\n",
      "        -2.9920e-02,  1.0278e-02, -3.5630e-02,  1.9940e-02,  2.0331e-02,\n",
      "        -1.3838e-03, -1.7528e-03, -2.6401e-03, -1.0210e-02, -3.2785e-02,\n",
      "         2.2378e-04, -3.0125e-03,  1.4184e-03, -4.0632e-02, -5.9104e-02,\n",
      "         1.9363e-02, -3.8822e-04,  1.9666e-02,  1.5105e-03, -2.0290e-02,\n",
      "         2.6886e-02, -1.4454e-02,  1.2990e-02,  8.5418e-03, -4.8611e-02,\n",
      "        -5.5131e-03, -3.8277e-02,  7.7857e-02,  1.6384e-03, -3.6197e-03,\n",
      "        -3.2984e-02,  2.6179e-02, -1.1904e-02,  1.8972e-02,  4.7659e-03,\n",
      "        -1.1946e-02,  8.4609e-04, -6.2873e-02,  9.7024e-03,  1.1826e-02,\n",
      "        -3.1838e-03,  1.3531e-02,  3.3752e-03,  3.6891e-02, -2.9675e-02,\n",
      "         2.2803e-03, -1.0677e-02, -9.6820e-03,  2.8454e-02,  2.8928e-03,\n",
      "         4.8233e-03,  1.4947e-02,  1.2134e-02, -1.7028e-02, -1.6649e-02,\n",
      "         1.3431e-02, -1.0592e-02, -3.8583e-03,  1.6301e-03,  2.9242e-02,\n",
      "        -9.8895e-03, -3.0052e-03,  2.4292e-02, -2.1945e-02,  8.0480e-03,\n",
      "        -5.3493e-04,  3.2889e-02,  5.6940e-03, -5.5457e-03,  9.9390e-03,\n",
      "        -2.7421e-03,  1.1367e-02, -1.9153e-02,  1.4339e-02,  2.1613e-02,\n",
      "        -1.4027e-02,  4.5918e-03,  1.3898e-02, -1.6853e-02, -1.8225e-02,\n",
      "         1.2514e-03,  9.6474e-03, -1.5422e-02,  1.3031e-02,  9.8926e-03,\n",
      "        -1.5265e-02, -6.9646e-04,  1.8242e-02, -2.2976e-02,  1.0677e-02,\n",
      "        -5.9094e-03, -1.1955e-03, -1.5896e-03, -5.3631e-03, -1.6244e-02,\n",
      "        -2.7899e-02,  3.1247e-02, -8.6326e-03, -1.6680e-02,  4.7067e-03,\n",
      "         1.9092e-02, -5.7089e-04, -2.0938e-02, -7.5476e-03,  5.0905e-03,\n",
      "        -9.5115e-03,  1.1550e-02,  6.4566e-03, -1.6832e-02, -6.8547e-03,\n",
      "         2.2062e-03,  1.2252e-02, -1.1757e-02,  1.7154e-02, -1.9841e-02,\n",
      "        -7.6795e-02,  4.4356e-02, -1.8524e-02, -2.0824e-02, -4.5741e-02,\n",
      "         1.0627e-02, -1.1999e-03,  3.8791e-02, -3.3901e-02,  1.8271e-02,\n",
      "        -2.2141e-02,  1.3657e-02,  4.6621e-03, -8.5101e-03, -1.5513e-02,\n",
      "         2.2870e-02, -1.1815e-02, -6.4941e-02, -2.1801e-03,  3.3460e-02,\n",
      "         1.7392e-02, -3.8943e-02, -4.9047e-02, -4.6534e-03,  4.6398e-02,\n",
      "        -4.8279e-02, -1.0184e-02,  5.6885e-02, -8.9877e-03])), ('bert.encoder.layer.0.attention.output.dense.weight', tensor([[ 0.0083,  0.0139, -0.0062,  ...,  0.0037, -0.0027, -0.0114],\n",
      "        [ 0.0097, -0.0259,  0.0145,  ..., -0.0052,  0.0057,  0.0069],\n",
      "        [-0.0213, -0.0029, -0.0033,  ...,  0.0058, -0.0065,  0.0079],\n",
      "        ...,\n",
      "        [ 0.0345, -0.0302,  0.0196,  ...,  0.0526,  0.0366,  0.0165],\n",
      "        [ 0.0202, -0.0137,  0.0064,  ..., -0.0234, -0.0561, -0.0338],\n",
      "        [-0.0170,  0.0322, -0.0199,  ..., -0.0032,  0.0306,  0.0147]])), ('bert.encoder.layer.0.attention.output.dense.bias', tensor([-7.5950e-03, -2.7152e-03, -1.9151e-03,  4.0727e-03, -1.2763e-03,\n",
      "        -1.3086e-02,  2.0501e-04, -1.8547e-02, -6.8779e-03, -4.0671e-03,\n",
      "        -9.8096e-03, -2.2894e-02,  4.8414e-04, -2.3307e-02, -3.1445e-02,\n",
      "        -1.4990e-02, -2.8564e-02, -1.7759e-02, -1.3421e-02, -3.9264e-03,\n",
      "        -2.1170e-02, -4.3530e-02, -5.7570e-02, -1.2300e-03, -2.2766e-03,\n",
      "         7.3463e-03, -1.9881e-02, -2.4927e-02,  2.7999e-02, -8.6062e-02,\n",
      "        -3.2972e-02, -8.3437e-02, -6.2393e-02, -2.1495e-02,  2.2626e-02,\n",
      "        -7.5710e-02, -7.1166e-02, -1.1204e-01, -5.3425e-02, -1.6418e-03,\n",
      "        -5.0666e-02, -1.3921e-01, -5.1459e-02,  1.9595e-02, -1.9371e-02,\n",
      "        -1.4137e-01, -2.4922e-02, -4.5391e-02, -1.1033e-01,  1.5242e-02,\n",
      "        -2.1931e-02, -2.7955e-02,  4.8765e-03,  2.6002e-02, -3.7601e-02,\n",
      "        -5.2956e-02, -2.2607e-02, -6.1656e-02, -3.2410e-02, -4.4060e-02,\n",
      "         2.5250e-02, -8.1271e-02,  2.8940e-02, -5.8839e-02, -6.9024e-02,\n",
      "        -6.4525e-02,  2.9175e-02, -6.7025e-02,  5.8553e-02, -2.8996e-02,\n",
      "         4.5625e-03, -2.4523e-02,  5.6993e-02, -6.5117e-02, -1.0794e-01,\n",
      "         3.3080e-03,  4.4759e-02, -6.7020e-02, -1.8622e-03, -3.9695e-02,\n",
      "         1.0710e-01,  3.2600e-01,  8.5928e-02, -4.7808e-02,  1.3600e-03,\n",
      "        -3.6099e-02,  5.9829e-02,  1.0120e-02, -3.1428e-02,  1.0163e-01,\n",
      "         4.5694e-02, -2.7791e-02,  2.1566e-02,  7.2426e-02,  4.7279e-02,\n",
      "         5.0074e-02,  4.4079e-03,  1.7885e-02, -7.8829e-03,  2.2986e-02,\n",
      "         8.3359e-03,  1.6694e-01,  3.8507e-02,  1.5415e-02, -4.0575e-02,\n",
      "         2.0867e-01,  1.3098e-02,  4.8294e-02,  7.1248e-03, -3.2958e-03,\n",
      "        -9.6429e-04,  6.5927e-03, -1.4190e-02,  7.1173e-02,  5.0019e-02,\n",
      "         4.5997e-03,  3.6134e-02,  1.0008e-02,  2.0031e-02,  2.9212e-02,\n",
      "         2.0624e-02, -1.1495e-04,  3.7660e-02,  2.9640e-02,  1.3073e-02,\n",
      "        -5.5254e-02,  1.4106e-02,  5.9026e-02, -4.1109e-02,  1.7469e-02,\n",
      "         1.3530e-01,  3.0672e-02, -7.4181e-03,  5.8570e-03,  1.0291e-02,\n",
      "        -1.6660e-02, -2.8696e-02,  2.0078e-02, -3.4560e-02, -7.4026e-04,\n",
      "        -1.3482e-02,  2.8538e-02,  1.6269e-02,  4.6076e-02, -1.4380e-02,\n",
      "        -5.6582e-03, -4.0633e-02, -2.1100e-02, -2.0078e-02,  2.2114e-02,\n",
      "        -2.9125e-02, -2.3818e-02,  2.4510e-03,  8.2524e-03, -1.5256e-03,\n",
      "         3.9894e-02, -7.2905e-02,  3.0365e-02, -1.3977e-02,  3.3324e-01,\n",
      "        -9.2819e-03,  1.1114e-02, -5.6946e-02,  8.3112e-03,  1.0909e-02,\n",
      "         1.1796e-01, -3.6026e-02,  8.3115e-02, -6.2781e-03,  1.3069e-02,\n",
      "         2.3383e-02, -3.0457e-02, -2.7273e-02,  7.6732e-02, -2.7378e-02,\n",
      "         1.7595e-05, -4.6628e-02,  1.5698e-02, -5.5572e-04,  9.2330e-02,\n",
      "        -1.8878e-02,  1.4042e-01, -6.7424e-03,  2.1505e-03, -7.3668e-02,\n",
      "        -6.8414e-02, -2.3703e-02,  3.8831e-01,  2.1632e-02,  7.0960e-02,\n",
      "        -1.9604e-02,  1.7341e-02,  1.0861e-02,  4.7669e-02, -9.1220e-02,\n",
      "         2.0712e-02,  1.8978e-02,  1.9853e-02, -1.4266e-02,  2.5993e-02,\n",
      "        -3.6804e-02,  3.2422e-02, -7.6895e-02,  2.9044e-02, -7.1736e-02,\n",
      "         3.9093e-02, -5.0257e-02,  3.6955e-02, -1.7037e-02,  2.0754e-02,\n",
      "        -1.0151e-02, -1.8700e-02, -1.8867e-02, -1.2514e-03,  1.3582e-02,\n",
      "         5.0994e-02, -2.2908e-02,  4.2830e-02, -8.6210e-02,  5.3304e-03,\n",
      "        -2.6513e-02,  6.5557e-03, -2.6874e-01,  1.0713e-02, -1.0844e-01,\n",
      "         6.6644e-02, -3.5377e-02,  3.4261e-02,  1.4682e-02,  9.6790e-03,\n",
      "        -9.0319e-02, -1.3018e-02, -8.1876e-03,  4.7659e-02, -3.6608e-04,\n",
      "         2.0411e-02, -1.9068e-01, -1.0019e-02, -5.4138e-02,  2.5872e-02,\n",
      "        -4.0986e-02, -7.5314e-03,  1.1353e-02, -3.2270e-03,  2.6538e-02,\n",
      "         1.4301e-01, -9.4519e-02, -5.6824e-02, -2.9522e-03,  4.3174e-03,\n",
      "        -4.0185e-03,  3.5953e-02, -3.5733e-02, -6.1990e-02, -2.4452e-03,\n",
      "         4.8947e-02,  6.2508e-03,  4.9595e-03, -1.1366e-01, -1.0655e-02,\n",
      "         6.3344e-03, -1.2478e-03, -1.6788e-02, -3.2916e-02,  9.5778e-03,\n",
      "         3.9068e-02,  2.0420e-02,  2.0601e-02, -6.0200e-02,  4.1173e-02,\n",
      "        -2.7266e-02,  6.3256e-02, -2.7824e-02,  3.6748e-03, -8.0638e-02,\n",
      "         8.6023e-02, -2.9915e-02,  5.9519e-02, -1.2152e-02,  9.3398e-02,\n",
      "        -4.1882e-02, -1.9018e-02, -5.1397e-02,  1.5243e-02, -5.0361e-02,\n",
      "         5.4948e-02, -2.3241e-02,  5.5965e-02, -4.1890e-02,  7.5069e-03,\n",
      "        -3.3819e-02,  6.3274e-02, -3.6822e-02,  2.6647e-02, -2.7131e-02,\n",
      "         1.9342e-02, -1.9143e-02,  1.7305e-02, -4.3848e-02,  8.5386e-02,\n",
      "        -3.6475e-02, -7.0843e-03,  1.5950e-02,  3.6692e-02, -1.6479e-02,\n",
      "        -9.2348e-03, -5.8904e-02,  1.0250e-01, -5.2529e-02,  5.6064e-02,\n",
      "        -5.4356e-02, -5.0668e-02, -1.4519e-02,  1.1726e-02, -3.3775e-02,\n",
      "         2.2752e-02, -4.6840e-02, -8.0026e-03,  9.9145e-03,  1.9535e-03,\n",
      "         3.3990e-03, -8.5282e-03, -5.4019e-02,  3.2363e-02, -4.9636e-02,\n",
      "         3.1031e-02, -6.0958e-03,  6.0243e-03,  1.7335e-02,  5.4513e-02,\n",
      "        -2.8727e-04,  7.9900e-02,  5.1314e-03, -1.5881e-02, -5.7723e-02,\n",
      "         4.3441e-02,  2.1705e-02,  2.7503e-02, -6.8771e-03,  3.4382e-02,\n",
      "        -2.3544e-02,  4.7836e-02,  1.4689e-02,  7.6625e-02, -6.7307e-02,\n",
      "         1.8229e-02, -4.1367e-02, -2.2511e-02, -1.6939e-01,  6.1731e-02,\n",
      "        -9.2287e-02,  7.7810e-03,  4.3406e-03,  6.9771e-02, -4.3650e-02,\n",
      "         8.4106e-02, -4.1946e-02,  7.1023e-02, -1.2907e-02,  7.1539e-02,\n",
      "         5.7849e-03, -3.8630e-02, -2.4707e-02, -6.2605e-02, -1.0182e-03,\n",
      "        -6.9971e-02,  1.0671e-03,  7.6882e-02, -2.7558e-02,  1.0830e-01,\n",
      "        -4.4960e-03,  9.9588e-03, -2.5758e-02,  6.6265e-04,  5.2175e-03,\n",
      "        -2.1006e-02, -2.9079e-03,  3.4482e-02, -3.4105e-02,  5.4736e-02,\n",
      "        -5.3043e-03, -5.0339e-03, -1.1390e-02,  4.3307e-02])), ('bert.encoder.layer.0.attention.output.LayerNorm.weight', tensor([1.2276, 1.4581, 1.3380, 1.5997, 1.3334, 1.2644, 1.3334, 1.3172, 1.4123,\n",
      "        1.2865, 1.1905, 1.4074, 1.3412, 1.1894, 1.2153, 1.1124, 0.8673, 1.1744,\n",
      "        1.1488, 1.0504, 1.0858, 1.1190, 0.9614, 1.1642, 1.4623, 1.2818, 1.2399,\n",
      "        1.1006, 0.8811, 0.8868, 0.8467, 0.8431, 0.9498, 0.9297, 0.8675, 1.0012,\n",
      "        1.0642, 1.0548, 1.1443, 1.0654, 1.0479, 0.8615, 0.9239, 0.9698, 0.8521,\n",
      "        0.8339, 0.8960, 0.8770, 0.9766, 0.9674, 0.9802, 1.0084, 1.1646, 0.9678,\n",
      "        1.0592, 0.9882, 1.0135, 0.9693, 0.9221, 0.9337, 0.9341, 0.9068, 0.9224,\n",
      "        0.8775, 0.9117, 0.8606, 0.9472, 0.9605, 0.9424, 0.9648, 0.9106, 0.9442,\n",
      "        0.8914, 0.9148, 0.9819, 0.9402, 0.9244, 0.9194, 0.8987, 0.9334, 1.0468,\n",
      "        1.2590, 1.0066, 0.9551, 0.9908, 0.8806, 0.9373, 0.9502, 1.0128, 0.9932,\n",
      "        0.9443, 0.9292, 0.9475, 0.9353, 0.9603, 0.9200, 0.9481, 0.9490, 0.9954,\n",
      "        0.9677, 0.9721, 0.7987, 0.9523, 0.9712, 0.9790, 1.0322, 0.9506, 0.9497,\n",
      "        0.9789, 0.9179, 0.9001, 0.9592, 1.0197, 0.9692, 0.9810, 0.9713, 0.9685,\n",
      "        0.9859, 0.9833, 0.7712, 1.0875, 1.0037, 0.9367, 0.9727, 0.9648, 1.0064,\n",
      "        0.9988, 0.9442, 0.9328, 0.9656, 0.9205, 0.9669, 0.9272, 0.9953, 1.0007,\n",
      "        0.9606, 0.9770, 0.9923, 0.9853, 0.9811, 0.9606, 0.9782, 0.9923, 0.9933,\n",
      "        0.9862, 1.0036, 0.9615, 0.9556, 0.9754, 0.9807, 0.9973, 0.9476, 1.0064,\n",
      "        0.9418, 0.8391, 0.9591, 0.9165, 0.9847, 0.9637, 1.2362, 0.9945, 0.9765,\n",
      "        0.9897, 0.9681, 0.9568, 1.0076, 1.0203, 0.9795, 0.9193, 0.9948, 0.9688,\n",
      "        0.9253, 0.9641, 0.9623, 1.0300, 0.9879, 0.9948, 0.9387, 0.8247, 0.9384,\n",
      "        0.9838, 0.8261, 0.9576, 0.9471, 1.0208, 1.0097, 0.9697, 1.8536, 1.0062,\n",
      "        0.9670, 0.9274, 1.0079, 0.9754, 0.9802, 1.0026, 0.8929, 0.9586, 0.9954,\n",
      "        0.9908, 0.9532, 1.0081, 0.9922, 0.9776, 0.9766, 1.0042, 0.9600, 0.9113,\n",
      "        0.9561, 0.9849, 0.9941, 0.9384, 0.9850, 0.9959, 0.9650, 0.9595, 0.9637,\n",
      "        0.9740, 0.9367, 0.8835, 0.9604, 0.9587, 0.9647, 0.8769, 0.9665, 0.9689,\n",
      "        0.9681, 0.9850, 1.0203, 1.0021, 0.9805, 0.9831, 1.0036, 0.9627, 0.9393,\n",
      "        0.9307, 0.9686, 0.8720, 0.9731, 0.9595, 0.9780, 0.9651, 0.9402, 0.9661,\n",
      "        0.9782, 1.0022, 0.9957, 0.9865, 0.8384, 0.9557, 0.9615, 0.9759, 0.9493,\n",
      "        1.0089, 1.0493, 0.9704, 0.9789, 0.9727, 0.9405, 0.9450, 0.9585, 0.9517,\n",
      "        0.9821, 0.9882, 1.0020, 0.9295, 0.9642, 0.9517, 1.0192, 0.9442, 0.9703,\n",
      "        1.0128, 0.9306, 0.9922, 0.9600, 0.9229, 0.9474, 1.0234, 0.9303, 0.9686,\n",
      "        0.9285, 0.9913, 0.9844, 0.9808, 1.0207, 1.0205, 0.9629, 0.9972, 0.7435,\n",
      "        0.9972, 0.9896, 0.9903, 0.9824, 0.9600, 0.9723, 0.8921, 0.9887, 1.0037,\n",
      "        0.9905, 1.0195, 0.9845, 0.9814, 0.9544, 1.0009, 0.9799, 0.9974, 0.9367,\n",
      "        0.9901, 0.9940, 0.9414, 0.9509, 0.9361, 0.8373, 0.9689, 0.9841, 0.9712,\n",
      "        0.9663, 0.9929, 1.0111, 0.9441, 0.9803, 0.9832, 0.9662, 1.0177, 0.9680,\n",
      "        0.9731, 1.0676, 0.9407, 0.9787, 0.9958, 0.9518, 0.9926, 0.9726, 0.9910,\n",
      "        0.7679, 0.8763, 0.9407, 0.9699, 0.9049, 0.9733, 0.9743, 1.0326, 0.9653,\n",
      "        0.9559, 0.9689, 0.9460, 0.9837, 0.9623, 0.9325, 0.9736, 0.9656, 0.9846,\n",
      "        0.9260, 0.9533, 0.9618, 0.9796, 1.0094, 1.0112, 0.9493, 0.9565, 0.9422,\n",
      "        0.9365, 0.9265, 0.9868, 0.8343, 0.9661, 0.9757, 0.9874, 0.9820, 0.9752,\n",
      "        0.9162, 0.9530, 0.9583, 0.9919, 0.9780, 0.9624, 0.8935, 0.9583, 0.9160,\n",
      "        0.9447, 0.9579, 0.9370, 0.9684, 1.0363, 0.9711])), ('bert.encoder.layer.0.attention.output.LayerNorm.bias', tensor([-1.3590e-01, -7.9125e-03, -4.2204e-02, -2.2406e-02,  1.6745e-02,\n",
      "        -1.0885e-01,  6.6511e-03, -9.7077e-02, -6.9840e-02, -1.1868e-01,\n",
      "        -5.6860e-02, -5.1182e-02, -5.3478e-02, -1.7725e-01, -1.0243e-01,\n",
      "        -1.3705e-01, -1.4763e-01, -1.4124e-01, -6.4074e-02, -7.6092e-02,\n",
      "        -1.6800e-01, -2.4694e-01, -2.2328e-01,  1.0787e-02,  2.5768e-02,\n",
      "         6.5205e-02, -1.0701e-01, -9.4313e-02,  9.8321e-03, -3.4866e-01,\n",
      "        -1.7653e-01, -2.8131e-01, -1.4356e-01, -6.2945e-02, -3.6800e-03,\n",
      "        -1.5242e-01, -1.9193e-01, -2.7016e-01, -1.1870e-01, -3.4535e-02,\n",
      "        -1.1325e-01, -4.4708e-01, -1.8592e-01,  1.7643e-02, -6.1736e-02,\n",
      "        -4.1831e-01, -2.3432e-02, -8.8426e-02, -3.0638e-01,  8.3406e-02,\n",
      "        -2.8387e-02, -5.6559e-02,  5.0615e-02,  7.3902e-02, -4.0949e-02,\n",
      "        -2.0030e-01, -1.5118e-01, -1.5784e-01, -8.1105e-02, -8.5092e-02,\n",
      "        -1.3467e-03, -2.1763e-01,  8.5132e-02, -5.3449e-02, -1.8885e-01,\n",
      "        -1.4420e-01,  1.4409e-01, -1.7021e-01,  2.5561e-01,  1.4962e-02,\n",
      "         6.0702e-02, -8.8950e-02,  2.5792e-01, -1.4816e-01, -4.3322e-01,\n",
      "         9.6638e-02,  2.1836e-01, -1.0596e-01,  1.7608e-02, -3.6113e-02,\n",
      "         4.0108e-01,  9.6994e-01,  2.6570e-01, -3.2281e-02, -2.3248e-02,\n",
      "        -1.2081e-01,  1.6587e-01,  1.7983e-01, -2.3156e-01,  4.6230e-01,\n",
      "         6.2863e-02, -1.0300e-01,  1.1305e-03,  2.4776e-01,  1.6582e-01,\n",
      "         2.2142e-01, -1.7544e-02,  9.9232e-02, -1.5604e-01,  1.3052e-01,\n",
      "        -7.6265e-02,  4.3782e-01,  5.0402e-02,  5.6141e-02, -1.2535e-01,\n",
      "         5.6929e-01, -1.0478e-02,  2.1829e-01, -1.6179e-02,  1.2693e-01,\n",
      "        -1.0601e-02,  4.0101e-02, -1.6102e-01,  3.3587e-01,  1.6933e-01,\n",
      "        -9.0187e-02,  2.8973e-02,  2.7249e-02,  4.0210e-02,  1.0864e-02,\n",
      "         1.0272e-01,  2.2699e-02,  1.7438e-02,  6.8929e-02,  9.6761e-02,\n",
      "        -2.1044e-01, -4.6575e-03,  2.2444e-01, -1.4199e-01,  7.7076e-02,\n",
      "         3.6885e-01,  1.4110e-01, -3.9620e-02, -5.6086e-03, -9.6164e-03,\n",
      "        -7.6943e-03, -6.0879e-02,  5.1893e-02, -6.4261e-02,  2.0807e-02,\n",
      "        -6.4790e-02,  1.0078e-01,  4.5740e-03,  1.1479e-01, -3.8962e-02,\n",
      "         3.1736e-03, -2.5500e-01, -4.1035e-02, -1.0887e-01,  9.5536e-02,\n",
      "        -6.5668e-02, -4.9629e-02, -5.2561e-02,  3.5801e-03, -3.9217e-03,\n",
      "         1.1813e-01, -2.9583e-01,  2.3454e-01, -7.1874e-02,  1.0772e+00,\n",
      "        -9.5043e-02,  4.0944e-02, -2.1245e-01,  1.0096e-01, -2.3898e-02,\n",
      "         4.3968e-01, -1.5746e-01,  4.1938e-01, -4.5672e-02,  1.0984e-01,\n",
      "         1.3443e-01, -1.2987e-01, -1.3384e-01,  2.4476e-01, -1.0421e-01,\n",
      "         7.9548e-03, -7.7763e-02,  6.1047e-02, -9.0798e-02,  3.3146e-01,\n",
      "        -1.8922e-01,  3.8862e-01, -1.8493e-02,  5.3612e-02, -3.1642e-01,\n",
      "        -1.9027e-01, -4.6987e-02,  1.5418e+00,  1.0613e-01,  3.3489e-01,\n",
      "        -4.4021e-02,  1.0994e-01, -9.2511e-02,  1.4112e-01, -3.1800e-01,\n",
      "         1.5890e-01,  1.1671e-01,  4.2429e-02,  1.5363e-02,  1.0990e-01,\n",
      "        -1.7041e-01,  1.0134e-01, -2.1301e-01,  1.3942e-01, -2.6483e-01,\n",
      "         1.1005e-01, -1.1510e-01,  1.0212e-01, -7.1820e-02,  1.2965e-01,\n",
      "        -5.7925e-02, -6.1665e-02, -1.4944e-01, -4.4571e-02, -3.7012e-02,\n",
      "         8.7526e-02, -4.4248e-02,  4.5239e-02, -3.5684e-01, -2.9992e-02,\n",
      "        -1.7891e-01, -8.9787e-02, -6.9136e-01,  7.4499e-03, -3.7942e-01,\n",
      "         2.1962e-01, -7.5661e-02,  1.5696e-01,  8.6694e-02,  5.2050e-03,\n",
      "        -1.9972e-01, -1.7521e-02, -1.6899e-01,  2.1136e-01, -8.1199e-02,\n",
      "         2.4703e-02, -4.9964e-01, -1.6780e-02, -1.4769e-01,  1.7653e-02,\n",
      "        -1.0513e-01,  3.6609e-02,  2.9031e-02,  1.0359e-01,  4.2666e-02,\n",
      "         4.8079e-01, -3.9585e-01, -1.0752e-01, -5.5077e-03, -2.5218e-03,\n",
      "        -2.9229e-02,  1.4983e-01, -1.3784e-01, -2.4494e-01,  2.1480e-02,\n",
      "         1.0124e-01,  4.1114e-02,  9.1133e-02, -4.1654e-01, -5.7247e-02,\n",
      "         3.1712e-02, -5.7788e-02, -8.6451e-02, -1.4380e-02, -3.2737e-03,\n",
      "         1.4826e-01,  5.2097e-02,  1.2221e-01, -1.9758e-01,  8.8151e-02,\n",
      "        -7.7518e-02,  2.8670e-01, -1.9346e-01,  3.9423e-02, -1.9261e-01,\n",
      "         2.4775e-01, -1.6834e-01,  2.0544e-01, -6.9811e-02,  3.3936e-01,\n",
      "        -8.5693e-02, -4.5657e-02, -1.2381e-01,  6.4078e-02, -1.5364e-01,\n",
      "         1.6806e-01, -1.0083e-01,  9.2084e-02, -1.3545e-01, -3.3870e-02,\n",
      "        -1.2830e-01,  7.5333e-02, -1.5791e-01,  1.1245e-01, -1.9504e-01,\n",
      "         3.6584e-02, -3.5535e-02,  6.0493e-02, -2.0697e-01,  2.9653e-01,\n",
      "        -2.3177e-01, -6.8583e-02,  1.3935e-02,  1.3409e-01, -1.7618e-01,\n",
      "        -1.3047e-02, -1.4564e-01,  3.8148e-01, -1.6449e-01,  1.5591e-01,\n",
      "        -1.7801e-01, -2.3257e-01, -3.7189e-02, -6.7466e-02, -4.5276e-02,\n",
      "         8.0847e-03, -5.0502e-02,  6.8352e-02,  6.8717e-02,  7.3813e-02,\n",
      "         4.3673e-02, -1.3428e-02, -2.6412e-01,  2.2737e-01, -6.8414e-02,\n",
      "         1.7528e-01, -1.5074e-01, -8.2571e-02,  4.5960e-02,  2.0072e-01,\n",
      "        -6.4702e-02,  3.6292e-01, -3.7599e-02,  6.3711e-03, -2.4892e-01,\n",
      "         9.4283e-02,  7.5402e-02,  7.3496e-02, -3.2176e-02,  8.7833e-02,\n",
      "        -5.0071e-02,  1.8102e-01, -7.6324e-02,  2.0430e-01, -2.8240e-01,\n",
      "         3.8922e-02, -2.7958e-02,  1.4363e-02, -5.2403e-01,  1.9164e-01,\n",
      "        -3.3299e-01,  1.0388e-02, -2.3769e-02,  2.0361e-01, -9.7942e-02,\n",
      "         3.1383e-01, -1.6622e-01,  2.2622e-01,  2.0669e-02,  2.5336e-01,\n",
      "         3.1183e-02, -2.0391e-01, -9.5713e-02, -1.6107e-01, -2.7818e-02,\n",
      "        -2.7507e-01,  1.9545e-02,  2.8515e-01, -9.1068e-02,  3.8771e-01,\n",
      "        -1.8263e-01, -3.6194e-02, -8.1839e-02, -3.8237e-02,  3.8726e-03,\n",
      "        -5.5550e-02,  1.0228e-02,  2.0789e-01, -1.3127e-01,  2.1251e-01,\n",
      "        -3.5394e-02, -1.1148e-01, -1.0356e-01,  1.0656e-01])), ('bert.encoder.layer.0.intermediate.dense.weight', tensor([[ 0.0353, -0.0816, -0.0078,  ...,  0.0469, -0.0388,  0.0101],\n",
      "        [ 0.0579, -0.0885, -0.0428,  ...,  0.0873, -0.0205,  0.0104],\n",
      "        [-0.0080,  0.0186,  0.0087,  ...,  0.0251, -0.0046, -0.0224],\n",
      "        ...,\n",
      "        [ 0.0041,  0.0038, -0.0045,  ..., -0.0562, -0.0063,  0.0200],\n",
      "        [ 0.0111,  0.0106,  0.0553,  ..., -0.0153,  0.0137,  0.0192],\n",
      "        [-0.2011, -0.1383,  0.0157,  ...,  0.0004,  0.0369, -0.0389]])), ('bert.encoder.layer.0.intermediate.dense.bias', tensor([-0.0771, -0.1711, -0.0329,  ..., -0.0448, -0.0682, -0.1748])), ('bert.encoder.layer.0.output.dense.weight', tensor([[ 0.0996,  0.0013,  0.0415,  ...,  0.0188, -0.0511, -0.3211],\n",
      "        [ 0.0229,  0.1119, -0.0623,  ..., -0.0491, -0.0722, -0.1040],\n",
      "        [ 0.0376, -0.1275, -0.1452,  ..., -0.0156,  0.0021,  0.0334],\n",
      "        ...,\n",
      "        [ 0.0021, -0.0045,  0.0105,  ...,  0.0218, -0.0129, -0.0131],\n",
      "        [ 0.0047, -0.0072,  0.0147,  ..., -0.0484,  0.0031,  0.0099],\n",
      "        [-0.0074,  0.0211, -0.0040,  ..., -0.0161, -0.0182,  0.0024]])), ('bert.encoder.layer.0.output.dense.bias', tensor([-2.2052e-02, -2.4648e-02, -1.1778e-03, -2.9269e-02, -3.0558e-03,\n",
      "        -2.1906e-02, -2.3124e-02, -2.2825e-02, -5.4217e-02, -4.0801e-02,\n",
      "        -7.7560e-02, -4.1069e-02,  6.0429e-02, -7.4788e-02, -2.3442e-02,\n",
      "         1.7754e-03,  1.6489e-02,  6.9119e-03, -1.4135e-02, -1.1401e-02,\n",
      "        -2.4958e-02,  7.0991e-03, -2.1258e-02,  5.2993e-02,  2.3453e-02,\n",
      "         2.0685e-02, -9.8364e-03,  2.8136e-02, -5.7938e-03, -1.9358e-02,\n",
      "         2.1259e-02,  3.7035e-02, -1.4008e-02,  2.1001e-02, -1.5365e-02,\n",
      "         1.3698e-02,  1.7849e-02,  1.4402e-02, -1.1721e-02,  1.0683e-02,\n",
      "        -1.0953e-02, -9.2631e-02,  1.1071e-02, -2.4571e-02,  2.0591e-02,\n",
      "        -2.8825e-02, -2.2987e-02,  1.6680e-02, -3.1330e-02,  4.6268e-03,\n",
      "         5.3555e-03,  3.5737e-02, -2.6637e-03,  5.7529e-03, -7.1902e-03,\n",
      "        -5.1316e-03, -3.1845e-02, -6.1682e-03, -1.6539e-02, -5.1562e-03,\n",
      "        -1.6588e-02, -3.8879e-02, -5.1116e-03,  2.3655e-02,  3.4109e-03,\n",
      "        -3.7235e-02, -7.3251e-03, -1.8209e-03,  1.4358e-02,  1.1122e-02,\n",
      "        -1.6302e-03, -1.2177e-03,  2.5919e-02,  4.4890e-03, -1.6945e-02,\n",
      "         1.7945e-02,  4.5663e-02, -2.5489e-02, -2.3233e-02, -1.5176e-02,\n",
      "        -1.8108e-02,  1.6389e-01,  3.0026e-02,  1.6008e-02,  2.0483e-02,\n",
      "         1.3399e-02,  3.2103e-02,  7.1913e-02,  2.0603e-02,  4.1344e-02,\n",
      "         1.0259e-02, -3.5083e-02,  1.2670e-02, -9.1739e-03, -8.5090e-03,\n",
      "         4.0846e-02, -6.6230e-03, -3.1970e-03, -1.9673e-02, -5.3823e-03,\n",
      "         2.0122e-02,  1.8281e-03, -5.7987e-03, -1.5781e-02,  1.0519e-02,\n",
      "        -4.1223e-02,  8.5411e-03,  6.4246e-03, -1.6886e-02,  2.5590e-02,\n",
      "        -2.9170e-02, -1.6741e-02, -5.6889e-02,  6.6049e-02,  3.5385e-02,\n",
      "        -1.5870e-03, -1.5576e-03,  1.5205e-02, -2.2546e-03, -9.4545e-03,\n",
      "         2.9199e-02,  9.4173e-04,  3.5315e-02,  6.2710e-02, -1.8021e-02,\n",
      "        -1.5811e-02,  9.8380e-03,  1.8905e-02,  5.2702e-03, -2.9430e-03,\n",
      "         1.3901e-02, -1.3429e-03,  8.4494e-03,  1.0134e-02,  2.4550e-03,\n",
      "         1.3299e-02, -2.7255e-03,  4.8867e-03, -2.6104e-02,  6.7039e-03,\n",
      "        -2.8414e-02,  2.2525e-02,  2.0879e-02, -5.0151e-03,  2.4241e-03,\n",
      "         5.4097e-03, -5.4662e-02, -3.5631e-02, -4.4306e-02,  1.1618e-02,\n",
      "        -2.3485e-02,  2.7030e-02, -2.1775e-02, -4.0741e-02, -2.1548e-03,\n",
      "        -7.7118e-03, -4.5082e-02,  1.2204e-02, -4.4432e-02,  1.6659e-01,\n",
      "        -1.9573e-02, -2.8705e-02, -1.2918e-02,  1.4809e-02,  8.1215e-03,\n",
      "         1.2634e-02, -8.9684e-04, -3.7086e-02, -4.0212e-02,  2.3115e-02,\n",
      "         4.8971e-02, -1.3999e-02, -5.1342e-03,  3.5910e-02,  7.5425e-04,\n",
      "         3.1037e-02, -3.4821e-02,  4.6966e-03,  1.6552e-02,  7.1894e-02,\n",
      "        -1.4180e-02, -3.1511e-02,  6.8457e-03, -7.5538e-03, -1.5793e-02,\n",
      "        -2.4862e-02,  5.1736e-03,  9.2420e-02,  1.7805e-02,  5.9253e-02,\n",
      "        -6.1029e-02, -5.1903e-03, -2.7708e-02,  2.8663e-02, -6.4268e-02,\n",
      "        -3.6355e-02, -9.5167e-03, -4.9773e-03, -3.6872e-03,  3.6223e-03,\n",
      "        -9.3606e-03,  4.2188e-05, -2.4843e-02,  1.5458e-02, -4.4239e-02,\n",
      "         1.4989e-02,  2.8912e-02,  1.8374e-02, -2.5441e-02,  5.2423e-02,\n",
      "        -6.0098e-03,  1.4286e-02, -2.3705e-03, -3.8726e-02, -1.1701e-02,\n",
      "         2.4109e-02, -1.0454e-02,  1.6277e-02, -2.6849e-02,  2.5200e-03,\n",
      "        -4.5014e-02, -2.3347e-02, -6.7666e-02,  2.4817e-04, -7.8263e-02,\n",
      "         3.1935e-02,  6.2971e-03,  1.4408e-02,  2.1839e-02, -3.5863e-03,\n",
      "         6.7066e-04, -2.0185e-02, -4.5895e-02,  5.7347e-02, -6.7622e-02,\n",
      "        -8.9417e-03, -1.3926e-02,  5.4538e-03,  6.1750e-03,  2.0358e-02,\n",
      "        -2.1353e-02,  2.3055e-02, -1.2262e-02,  2.3147e-03,  4.4486e-02,\n",
      "         8.5228e-03, -4.8796e-02,  1.5758e-02, -3.2119e-02,  2.2890e-02,\n",
      "        -2.2888e-02, -1.9914e-04, -9.7365e-03, -5.4411e-03, -1.8477e-02,\n",
      "         2.7115e-02, -1.1086e-03,  3.9631e-03, -3.6107e-02, -1.8280e-02,\n",
      "         4.7788e-04, -3.0990e-02, -2.4977e-02, -1.3289e-02, -1.2440e-02,\n",
      "         1.8817e-02,  1.3585e-02,  3.5600e-02, -2.9057e-03,  3.8956e-02,\n",
      "        -2.5785e-02,  5.7536e-02, -1.1330e-02,  8.9880e-03, -4.1960e-02,\n",
      "         7.3218e-02, -2.1158e-03,  2.0700e-02,  2.9353e-02,  2.3662e-02,\n",
      "        -2.5119e-02,  2.5743e-03,  1.3637e-02,  1.7122e-02, -5.7778e-03,\n",
      "        -3.4687e-02, -9.3452e-03,  5.1659e-02, -2.5757e-02, -1.9060e-02,\n",
      "        -8.5379e-03,  2.4766e-02, -1.3405e-02,  6.1265e-03, -4.4379e-02,\n",
      "         4.4377e-03, -1.6515e-03, -1.3785e-02, -3.7381e-02,  2.7446e-02,\n",
      "        -2.2374e-02, -7.5362e-03,  8.4186e-03,  3.3769e-02,  1.3263e-02,\n",
      "        -4.6305e-03, -2.7363e-02,  3.7277e-02, -4.0353e-02, -1.4718e-02,\n",
      "         3.5183e-03,  3.0259e-02, -1.5682e-02,  1.0670e-02, -2.9007e-04,\n",
      "         1.3025e-02, -1.3099e-03,  4.0126e-02,  7.0306e-03, -2.3991e-03,\n",
      "         2.6493e-02,  8.6291e-03, -4.5211e-02,  4.1955e-02, -1.6374e-02,\n",
      "         9.6536e-03, -3.9085e-03,  1.6746e-02, -9.3137e-03,  2.5248e-02,\n",
      "        -1.9340e-02,  1.7365e-02,  2.1050e-02,  5.1365e-02, -4.4395e-03,\n",
      "         5.1193e-02,  3.7038e-02,  1.5611e-02, -1.3298e-02, -7.7340e-03,\n",
      "        -6.7249e-03, -2.5214e-02,  9.7007e-03,  2.9211e-02, -1.6368e-03,\n",
      "        -2.7708e-02,  8.0251e-03, -2.9641e-02, -1.0605e-01, -4.9650e-03,\n",
      "        -3.5936e-02,  2.5388e-02, -1.7712e-02,  2.6060e-03, -3.4021e-02,\n",
      "         7.8214e-02, -4.2867e-02,  1.1581e-02, -1.7972e-02,  1.1019e-02,\n",
      "         9.6083e-03, -1.8037e-02, -4.5064e-02,  3.4768e-02, -5.2602e-03,\n",
      "        -3.7201e-02,  7.1963e-03,  2.5034e-02, -3.1231e-02, -2.6852e-02,\n",
      "        -3.1500e-02,  4.1782e-03, -1.3202e-02, -2.3660e-02, -1.1123e-02,\n",
      "         4.1421e-02, -4.3964e-02,  7.0524e-04, -1.1621e-02,  3.2859e-03,\n",
      "         2.9953e-02,  4.8195e-03, -3.9133e-02,  4.7620e-02])), ('bert.encoder.layer.0.output.LayerNorm.weight', tensor([0.4945, 0.9591, 0.7888, 0.8674, 0.8328, 0.6763, 0.6780, 0.7674, 0.9773,\n",
      "        0.6744, 0.4431, 0.8760, 0.7747, 0.6481, 0.6482, 0.5447, 0.5290, 0.6959,\n",
      "        0.6872, 0.6933, 0.7816, 0.7627, 0.5593, 0.8778, 1.0700, 0.9092, 0.9287,\n",
      "        0.8500, 0.6924, 0.5765, 0.5944, 0.6551, 0.8227, 0.7343, 0.5727, 0.8847,\n",
      "        0.8617, 0.7755, 0.9628, 0.9563, 0.8365, 0.4838, 0.6820, 0.8896, 0.4996,\n",
      "        0.3984, 0.7972, 0.6217, 0.5970, 0.9793, 1.0168, 0.9370, 1.1196, 0.9835,\n",
      "        0.9834, 1.0018, 1.0080, 0.8739, 0.8994, 0.9398, 0.9555, 0.9240, 0.9511,\n",
      "        0.9390, 0.9361, 0.9851, 1.0135, 1.0398, 1.0352, 1.0073, 1.0655, 1.0233,\n",
      "        1.0994, 0.9864, 0.8902, 1.0799, 1.0644, 1.1205, 1.0553, 1.0703, 0.8650,\n",
      "        0.4585, 1.0900, 1.0613, 1.1522, 0.9758, 1.0082, 1.1390, 0.9848, 0.9951,\n",
      "        1.1392, 1.1130, 1.1140, 0.9205, 1.1043, 1.0381, 1.1326, 1.0561, 1.1173,\n",
      "        1.1533, 1.1698, 0.8023, 1.1496, 1.1176, 1.0865, 0.3885, 1.1506, 1.0091,\n",
      "        1.2215, 1.1183, 1.1232, 1.1183, 1.1716, 1.1237, 1.1141, 1.1226, 1.1287,\n",
      "        1.1767, 1.1498, 0.8991, 1.1809, 1.1705, 1.1360, 1.1402, 1.1401, 1.0802,\n",
      "        1.1747, 0.9960, 1.0605, 1.1267, 0.7242, 1.0932, 1.0949, 1.1141, 1.1727,\n",
      "        1.1574, 1.1389, 1.1184, 1.1368, 1.1095, 1.1341, 1.0766, 1.2163, 1.1478,\n",
      "        1.1213, 1.1539, 1.1205, 1.1028, 1.1452, 1.1442, 1.1138, 1.1244, 1.1696,\n",
      "        1.1284, 0.8706, 1.1266, 1.0447, 0.9982, 1.0686, 0.5561, 1.1532, 1.1660,\n",
      "        1.0927, 1.2024, 1.1033, 0.9100, 1.1653, 0.9804, 0.9388, 1.1805, 1.0951,\n",
      "        1.0221, 1.0981, 1.1305, 1.1680, 1.1267, 1.2094, 1.1105, 1.0005, 0.9968,\n",
      "        1.1145, 0.7389, 1.1341, 1.0453, 1.0385, 1.0835, 1.1610, 0.8583, 1.1669,\n",
      "        1.1125, 1.1461, 1.1378, 1.0877, 1.0632, 1.0399, 1.0814, 1.1456, 1.1469,\n",
      "        1.1525, 1.1059, 1.1519, 1.1030, 1.0388, 1.1656, 1.1680, 1.1162, 0.9322,\n",
      "        1.1276, 1.1312, 1.1414, 1.1035, 1.1296, 1.1523, 1.1888, 1.1180, 1.1627,\n",
      "        1.1206, 1.0834, 0.8837, 1.1222, 1.1011, 1.1188, 0.6408, 1.1079, 1.1163,\n",
      "        1.1520, 1.1101, 1.0679, 1.1466, 1.1396, 1.0615, 1.1802, 1.1390, 1.1686,\n",
      "        1.1439, 1.1432, 0.8269, 1.1823, 1.0551, 1.1551, 1.0493, 1.1169, 1.1644,\n",
      "        1.1347, 1.2299, 0.9689, 1.0298, 0.9604, 1.1293, 1.1168, 1.1677, 1.0238,\n",
      "        1.1556, 1.0783, 1.1472, 1.0863, 1.1940, 1.1315, 0.9611, 1.1486, 1.1060,\n",
      "        1.1401, 1.1647, 1.1286, 1.1507, 1.1350, 1.1320, 1.1774, 1.0648, 1.1545,\n",
      "        1.2153, 1.0115, 1.1046, 1.1241, 1.1055, 1.1061, 1.1774, 1.0498, 1.1331,\n",
      "        1.0382, 1.1254, 1.1791, 1.1317, 0.9414, 1.1196, 1.1224, 1.1385, 0.6998,\n",
      "        1.1601, 1.1448, 1.1339, 1.1226, 1.1064, 1.1133, 1.0651, 1.1414, 1.1928,\n",
      "        1.1016, 1.1471, 0.9729, 1.0543, 1.1031, 1.1817, 1.1542, 1.1353, 1.1325,\n",
      "        1.1192, 1.0414, 1.1278, 1.0579, 1.0646, 0.9406, 1.1646, 1.0996, 1.0809,\n",
      "        1.1108, 1.1018, 1.1834, 1.0874, 1.1078, 1.1394, 1.1248, 1.1763, 1.1453,\n",
      "        1.1421, 1.1371, 1.0875, 1.1356, 1.1757, 1.0078, 1.1334, 1.0498, 1.1322,\n",
      "        0.9140, 0.8660, 1.1551, 1.1431, 1.0783, 1.1614, 1.1586, 1.1540, 1.1425,\n",
      "        1.1025, 1.1561, 1.0626, 1.1504, 1.1272, 1.1179, 1.0953, 1.1266, 1.0828,\n",
      "        1.0906, 1.0806, 0.9986, 1.1308, 1.1880, 1.1748, 1.0440, 1.0671, 1.0772,\n",
      "        1.0744, 1.0987, 1.1964, 0.6079, 1.1088, 1.0492, 1.1526, 1.0457, 1.0831,\n",
      "        0.8222, 1.0822, 1.0749, 1.1735, 1.1574, 1.1535, 1.0717, 1.1553, 1.0844,\n",
      "        1.1067, 1.0890, 0.9115, 1.1128, 1.1134, 1.1271])), ('bert.encoder.layer.0.output.LayerNorm.bias', tensor([ 1.2399e-02, -1.5386e-02, -2.3045e-02,  4.8608e-02,  3.5708e-02,\n",
      "        -5.9463e-02,  8.8892e-03,  3.5846e-03,  3.1275e-02,  4.7154e-03,\n",
      "         2.3685e-02,  1.8645e-02, -4.6252e-02,  2.5274e-02,  4.5551e-02,\n",
      "         1.2335e-01,  2.9062e-02,  1.4430e-02,  5.7023e-03, -5.6330e-03,\n",
      "         1.1673e-02,  6.9361e-02,  2.8334e-02,  2.1611e-02, -1.6522e-02,\n",
      "         3.4749e-02,  2.1426e-03,  1.5537e-02, -6.8356e-02,  6.9560e-03,\n",
      "        -1.2532e-02,  2.3882e-02,  5.5897e-02,  3.1176e-02,  2.1783e-02,\n",
      "        -3.3446e-02,  1.1437e-02,  3.4219e-02,  1.4280e-02,  1.4784e-03,\n",
      "        -2.4938e-02,  1.0013e-01,  5.8211e-02, -5.9492e-02,  4.7863e-04,\n",
      "         9.5438e-02, -5.0389e-02, -2.7789e-03,  7.4286e-02, -1.0452e-02,\n",
      "         1.7166e-03,  5.2329e-02,  6.2773e-03,  5.1365e-03,  4.2296e-03,\n",
      "        -6.0658e-03, -2.6373e-02, -4.2370e-02, -2.0186e-02, -2.9162e-02,\n",
      "        -3.6830e-02, -1.4677e-02, -1.2521e-02,  1.8719e-02,  1.4753e-02,\n",
      "        -1.0411e-01,  1.8896e-02,  4.8002e-02, -3.8122e-03, -3.2176e-02,\n",
      "        -3.4721e-02, -5.0796e-02, -5.3587e-02, -4.4082e-02, -2.2983e-02,\n",
      "        -7.2514e-02, -8.7688e-03, -6.7191e-02, -7.0642e-02, -8.3808e-02,\n",
      "        -1.4294e-01, -3.4220e-01,  1.8770e-02,  1.5234e-03,  4.1820e-02,\n",
      "        -2.6141e-02,  9.1010e-02,  1.0162e-01,  2.7971e-02, -6.8624e-02,\n",
      "         4.5222e-03, -1.1182e-02,  7.4233e-02, -5.6497e-02,  5.2543e-02,\n",
      "         3.4155e-02,  5.0321e-02,  2.9532e-02,  4.7954e-03, -2.6158e-02,\n",
      "        -3.3226e-02, -1.3483e-01,  1.5125e-02, -1.0962e-02,  4.3005e-02,\n",
      "        -1.9709e-01, -7.6241e-03, -5.7012e-02,  2.6412e-02,  3.0854e-02,\n",
      "         5.5184e-02,  5.8674e-02,  7.0199e-03,  8.6033e-02, -1.9134e-03,\n",
      "        -2.5169e-02,  4.8870e-04,  2.1162e-02, -7.6004e-05,  4.9938e-02,\n",
      "        -5.3696e-02,  3.8781e-02,  3.2034e-02, -2.8344e-02, -3.0891e-02,\n",
      "         4.3076e-02, -2.4870e-02,  3.8268e-02, -4.8939e-02,  7.5770e-02,\n",
      "        -1.5297e-01,  1.4131e-02, -1.4553e-02,  1.6116e-02,  2.8633e-02,\n",
      "         2.6246e-02,  4.0264e-02, -1.4447e-02, -5.6431e-03,  1.2465e-02,\n",
      "         5.3679e-03,  7.8984e-02, -3.8811e-02,  4.6905e-02,  1.3152e-02,\n",
      "        -3.2076e-02, -4.6283e-02,  7.0940e-02,  1.9454e-02,  4.2060e-03,\n",
      "         2.4642e-02,  5.8476e-02, -2.5090e-02,  1.6119e-02, -5.6113e-02,\n",
      "         3.3389e-02,  4.0749e-02,  2.2836e-02,  3.5227e-02, -8.0634e-02,\n",
      "         2.0917e-03,  2.7497e-03,  5.7320e-03,  8.9595e-02, -4.1227e-02,\n",
      "        -8.8144e-03,  3.2005e-03, -3.4858e-02,  5.1024e-02,  3.8976e-02,\n",
      "         3.8638e-02, -1.3721e-02, -1.1046e-02,  7.3268e-02,  3.0153e-03,\n",
      "         6.5631e-02,  2.9609e-02,  4.0858e-02, -2.6055e-02,  9.2096e-02,\n",
      "         9.4419e-03, -9.0203e-02, -1.4802e-03,  8.2011e-02, -6.8507e-04,\n",
      "         7.4118e-02, -1.1941e-02,  4.0613e-01,  4.4019e-02,  4.6963e-02,\n",
      "         9.3844e-03,  5.6830e-02,  1.6267e-02,  2.0610e-02, -5.9829e-02,\n",
      "         1.4097e-01,  2.6621e-02, -7.6197e-03,  5.1701e-02, -3.1413e-02,\n",
      "        -4.3066e-02,  3.6194e-02,  1.9072e-02, -2.6516e-02,  2.4130e-02,\n",
      "         4.1768e-02,  1.1310e-01,  7.5113e-02,  3.9400e-02, -3.1585e-03,\n",
      "        -7.9362e-03,  4.9904e-02, -5.2211e-02,  2.8061e-02, -6.7953e-02,\n",
      "         2.3384e-03,  2.7920e-02, -3.3750e-02, -1.3018e-02, -3.7587e-02,\n",
      "        -3.3029e-02, -4.6129e-02, -1.3261e-02,  1.5390e-02, -1.3999e-01,\n",
      "        -4.0531e-02,  1.9382e-02,  1.1200e-02,  2.9712e-02,  5.8213e-03,\n",
      "        -4.0452e-02,  5.2309e-02, -7.3719e-02,  6.9690e-03,  8.5428e-03,\n",
      "         1.8153e-02,  2.7184e-01, -1.2761e-02,  3.8142e-02,  2.9563e-03,\n",
      "        -3.0081e-02,  2.8915e-02,  2.8648e-03,  4.0343e-02, -3.7276e-02,\n",
      "         4.6089e-02, -4.7684e-02,  7.6496e-02, -5.2007e-02,  3.9840e-02,\n",
      "         3.5137e-03,  6.6327e-02, -7.9047e-02,  4.8730e-02,  2.8324e-02,\n",
      "         3.7644e-02,  1.0298e-02, -1.6126e-02, -2.0399e-02,  2.0896e-02,\n",
      "        -4.6158e-02,  7.5761e-02, -3.0961e-02,  1.1784e-02,  4.2281e-02,\n",
      "         5.5170e-02, -3.8771e-02,  1.5178e-02,  4.0011e-02, -8.6919e-03,\n",
      "         1.4212e-02,  1.3605e-01, -6.3167e-02,  5.3057e-04,  6.9951e-02,\n",
      "         9.4199e-02, -6.6421e-02,  8.0782e-02,  8.3895e-03,  9.2749e-03,\n",
      "        -2.3193e-02,  5.0301e-02,  8.8611e-03,  7.7182e-02,  7.9484e-03,\n",
      "         3.1718e-02, -3.7539e-02,  4.8090e-02, -2.1744e-02,  1.3176e-02,\n",
      "         3.4947e-02,  5.6730e-02, -5.8230e-02, -3.5707e-03, -2.0424e-02,\n",
      "         4.1240e-02, -2.9756e-02,  1.0164e-02, -3.3561e-02,  9.0866e-02,\n",
      "        -7.2581e-02,  3.5812e-02, -2.3668e-02, -2.0104e-02,  4.7841e-02,\n",
      "        -2.7513e-02, -3.4405e-02,  3.7801e-02,  2.1373e-02, -2.9347e-02,\n",
      "         1.8066e-02,  1.1458e-01,  3.1047e-02,  3.8359e-02,  8.6569e-03,\n",
      "         3.8964e-02, -4.7879e-02,  3.6027e-02,  3.4009e-02, -5.7107e-03,\n",
      "         1.6516e-02, -1.2109e-02,  8.1251e-02,  5.9300e-02,  1.7314e-02,\n",
      "         6.2302e-02, -2.9667e-02, -8.3265e-03,  9.3008e-03,  2.5398e-02,\n",
      "         1.3025e-02, -1.3620e-02, -2.8978e-02,  1.2308e-01, -6.4503e-03,\n",
      "         7.4254e-02, -3.1142e-02, -7.2926e-03, -3.2083e-02,  1.7981e-02,\n",
      "        -2.3827e-02, -2.0342e-02, -1.6684e-02,  1.3485e-02, -2.4871e-02,\n",
      "         1.8159e-02,  2.2705e-02,  3.2276e-02, -7.0714e-02,  8.1985e-02,\n",
      "        -1.9051e-02,  8.0389e-02,  5.9138e-02, -1.2617e-02,  3.2396e-02,\n",
      "         7.2323e-02, -1.7937e-02, -1.7693e-02, -3.2427e-02,  6.1613e-02,\n",
      "        -1.1036e-01,  8.4958e-02, -4.5130e-03,  6.5301e-02,  1.2153e-02,\n",
      "         3.9989e-02,  2.5379e-02,  2.9055e-02, -1.2603e-02, -3.7586e-02,\n",
      "        -1.3362e-01, -2.2969e-02, -3.1914e-04,  3.4515e-02,  4.2153e-02,\n",
      "         4.2557e-02,  6.7490e-02,  3.0501e-02, -3.1784e-02, -6.2080e-03,\n",
      "         1.8138e-02, -4.0623e-02, -3.8309e-03,  5.1781e-02])), ('bert.encoder.layer.1.attention.self.query.weight', tensor([[-0.1205, -0.2149,  0.1067,  ...,  0.0073,  0.0651,  0.0406],\n",
      "        [-0.0169,  0.0570, -0.2072,  ...,  0.0968,  0.0608, -0.0370],\n",
      "        [ 0.1027, -0.0143,  0.1237,  ..., -0.0206,  0.0048, -0.1096],\n",
      "        ...,\n",
      "        [-0.0505, -0.0408,  0.0881,  ..., -0.0333,  0.0126,  0.0252],\n",
      "        [-0.1231,  0.1048, -0.0848,  ..., -0.0535, -0.0534,  0.0609],\n",
      "        [-0.1024, -0.4379,  0.2613,  ..., -0.0382, -0.0108, -0.0367]])), ('bert.encoder.layer.1.attention.self.query.bias', tensor([ 4.0861e-03,  9.6629e-02, -2.1968e-01,  2.3978e-02,  2.1965e-01,\n",
      "        -1.8062e-03, -2.5772e-01, -1.8730e-01, -3.8561e-02, -8.6315e-02,\n",
      "        -1.7533e-01, -4.0168e-01,  2.2217e-01, -3.4583e-01, -1.4887e-02,\n",
      "        -2.8637e-01, -6.7083e-02,  3.2308e-01, -1.2707e-01,  3.5169e-01,\n",
      "        -5.6585e-03, -1.7457e-01,  2.0015e-01,  2.4993e-02,  3.0338e-02,\n",
      "        -1.6353e-01, -6.7835e-02, -2.6943e-01, -2.5776e-01, -1.7108e-02,\n",
      "        -1.6753e-01,  1.9199e-01, -3.3170e-03, -5.7386e-02, -3.8988e-03,\n",
      "        -7.1972e-02, -4.5114e-02, -4.2171e-02, -4.0262e-02,  4.9023e-02,\n",
      "        -8.2699e-02,  5.5754e-02, -1.6890e-02, -2.0626e-02,  1.4958e-02,\n",
      "        -5.2145e-02,  5.4950e-02,  1.2485e-01, -5.2698e-02,  7.5929e-02,\n",
      "        -5.4515e-02,  9.7832e-02, -1.5411e-02,  3.9964e-02,  8.0578e-02,\n",
      "         7.5324e-02,  4.0102e-02,  3.8236e-04, -7.9854e-02, -4.5858e-02,\n",
      "         1.3639e-02,  1.4806e-01,  7.7893e-02, -1.4800e-01,  1.3675e-01,\n",
      "        -4.2989e-02,  6.7854e-02,  9.2899e-02,  7.4720e-03,  1.1784e-01,\n",
      "        -1.5045e-01,  7.5016e-02,  6.4828e-02,  1.1248e-01, -1.4765e-01,\n",
      "        -1.9861e-02,  1.0298e-01,  1.9106e-01, -1.5425e-02, -1.3353e-02,\n",
      "         4.0165e-02, -1.3076e-01, -2.6082e-02,  1.7408e-01, -3.6029e-02,\n",
      "         8.1689e-02,  1.1645e-01,  4.1126e-02, -1.8541e-02,  1.3282e-03,\n",
      "         1.0362e-01, -3.3357e-02, -5.7785e-02,  1.4231e-01, -2.2561e-02,\n",
      "         6.8412e-02,  5.5714e-02, -3.0515e-01, -2.7408e-01, -4.2821e-01,\n",
      "         1.7951e-01, -1.2178e-01,  2.3075e-01, -2.1774e-01,  2.7332e-01,\n",
      "         1.3266e-01,  4.6249e-01, -4.8791e-02, -9.4383e-02, -3.0137e-01,\n",
      "        -4.8980e-02,  1.2582e-01,  3.1177e-01,  1.5325e-01,  2.8544e-01,\n",
      "        -1.9512e-01, -8.2205e-02, -1.7448e-01, -1.5451e-01, -2.0495e-02,\n",
      "        -4.1712e-01,  2.9739e-01,  5.3663e-02, -4.6892e-01, -4.2213e-02,\n",
      "         9.5231e-02,  3.8362e-01,  1.7384e-01, -5.3609e-02, -3.2468e-01,\n",
      "        -4.0725e-01,  4.0854e-02, -6.9194e-02,  8.1682e-02,  9.5859e-02,\n",
      "         5.7472e-02, -4.0671e-02, -2.5820e-01, -2.1967e-01,  3.5443e-01,\n",
      "        -3.3221e-01,  1.1463e-01,  2.7998e-01,  2.1177e-01,  1.7356e-01,\n",
      "        -8.8429e-02, -1.7545e-01,  1.4804e-01,  3.5183e-01, -7.9857e-02,\n",
      "        -3.8166e-01,  1.9760e-01, -1.5579e-01, -1.3607e-02, -3.4581e-01,\n",
      "        -2.5602e-01, -1.8949e-01,  2.0687e-02, -4.9256e-01,  1.3025e-01,\n",
      "         1.7948e-01,  1.5093e-01,  1.4143e-01, -2.1342e-01, -3.9578e-02,\n",
      "         2.0312e-01,  1.3786e-01, -5.8254e-02,  2.1852e-01,  1.2014e-02,\n",
      "         1.3684e-01, -1.7488e-01, -1.6468e-01,  1.5296e-02,  3.2753e-02,\n",
      "        -1.0619e-01,  1.2622e-01, -1.4715e-01, -2.0308e-01, -7.9487e-02,\n",
      "        -7.6139e-02, -1.4613e-02, -7.4510e-02,  1.1141e-01, -5.0933e-02,\n",
      "         2.4478e-01,  3.6011e-02,  9.1017e-02, -4.0028e-02,  3.2616e-02,\n",
      "        -5.4186e-02,  3.0746e-02,  7.3979e-02, -8.4731e-02, -1.3329e-01,\n",
      "         1.2251e-02,  1.1235e-01,  1.1237e-01,  1.3073e-01,  3.8203e-02,\n",
      "        -3.5983e-02,  1.3590e-02, -1.3745e-01, -9.1083e-02,  3.4969e-02,\n",
      "        -9.9593e-03, -3.0802e-03,  2.0043e-01,  1.0377e-02,  1.5113e-02,\n",
      "        -3.3279e-02, -1.7476e-02,  7.2872e-02,  5.0897e-02,  6.4499e-02,\n",
      "        -1.1066e-01, -8.4976e-02,  9.0244e-02, -6.2969e-02,  8.6209e-02,\n",
      "         7.3091e-02,  1.0178e-01, -2.9282e-02, -1.4137e-03, -8.4407e-02,\n",
      "        -3.5547e-02,  4.8075e-03,  1.0219e-01, -7.0994e-02,  9.1958e-02,\n",
      "         5.4398e-02, -1.4426e-02,  6.4457e-02, -1.2170e-02, -1.1236e-01,\n",
      "         3.6016e-02, -2.0094e-02, -6.2105e-02, -1.0583e-01, -4.9209e-02,\n",
      "        -1.1792e-02,  4.4419e-02,  5.6129e-03,  9.5730e-02, -1.6382e-02,\n",
      "        -3.2178e-02, -1.1997e-01,  6.8835e-02, -4.2325e-02,  2.2946e-02,\n",
      "        -2.7718e-02,  3.4153e-02,  1.0009e-02,  9.4609e-03, -7.3734e-04,\n",
      "         2.0334e-02,  3.0592e-01, -2.8556e-02,  3.4094e-01,  2.9874e-01,\n",
      "        -3.1704e-01,  1.5126e-01, -1.6891e-01,  3.4366e-01, -1.9267e-02,\n",
      "        -1.4185e-01, -1.6309e-01,  2.8361e-01,  4.7469e-01,  5.1413e-01,\n",
      "         4.7600e-04, -3.5040e-02,  1.0465e-01,  3.6595e-02, -2.4419e-02,\n",
      "        -1.6800e-01, -1.7864e-01,  1.3017e-01, -2.9564e-01, -3.4101e-01,\n",
      "        -3.4074e-01,  7.9093e-02,  3.7260e-01,  1.3385e-01,  1.2136e-01,\n",
      "        -3.2846e-02, -1.4885e-01, -7.2187e-03, -9.5121e-02,  7.6192e-02,\n",
      "        -2.2977e-02, -5.9771e-02,  8.8323e-02, -4.8425e-02, -3.1330e-02,\n",
      "         8.1555e-02,  1.0818e-01, -8.3653e-02, -1.2814e-02,  4.0536e-02,\n",
      "         3.3809e-02,  6.5540e-02,  6.1624e-02,  4.2631e-02,  5.5492e-02,\n",
      "         3.4892e-02, -7.7178e-03, -3.7763e-02,  4.6688e-02,  3.8884e-02,\n",
      "         4.8430e-02,  7.1110e-03,  2.4487e-02, -1.8310e-03,  4.3672e-02,\n",
      "         2.9320e-02,  5.5342e-02,  7.9302e-02, -4.3014e-02,  1.7633e-02,\n",
      "         6.1786e-02,  8.7605e-02,  3.0515e-01,  6.3754e-02,  7.1299e-02,\n",
      "         3.7422e-02,  5.9381e-02,  1.1390e-01,  3.4568e-01,  7.9957e-02,\n",
      "         1.8251e-01,  2.6013e-01,  5.0364e-02, -4.9725e-02,  9.4125e-02,\n",
      "         8.5882e-02,  3.7559e-02, -2.5823e-01,  1.2678e-01, -1.6304e-01,\n",
      "         3.6041e-02,  1.2620e-01,  2.5684e-01,  1.7657e-01,  7.2789e-02,\n",
      "         8.2525e-03,  8.9982e-02, -2.5140e-01,  2.1713e-01,  1.0088e-01,\n",
      "         1.4449e-01, -2.5646e-02, -3.4697e-01,  1.0227e-01, -6.1189e-02,\n",
      "        -1.4912e-01, -4.6906e-02,  2.1259e-02, -5.9230e-02, -1.9540e-01,\n",
      "         2.2131e-01, -1.7966e-02, -4.3078e-02,  1.3440e-03,  2.8477e-01,\n",
      "        -4.0591e-02,  1.8768e-02,  9.7466e-03,  2.1152e-01, -7.3308e-02,\n",
      "         5.0407e-02, -1.0824e-01, -2.4603e-02,  5.2533e-02,  1.6949e-01,\n",
      "        -1.0355e-01, -2.4686e-01,  1.6947e-01, -2.1288e-01,  1.0731e-01,\n",
      "        -9.2896e-03,  1.0036e-02,  9.8828e-03,  2.0484e-01])), ('bert.encoder.layer.1.attention.self.key.weight', tensor([[ 0.0011, -0.2119,  0.1609,  ..., -0.0068, -0.0114, -0.0068],\n",
      "        [-0.1859, -0.0264, -0.0861,  ...,  0.0327,  0.0129,  0.0345],\n",
      "        [ 0.0105,  0.1672,  0.0271,  ..., -0.0627,  0.0301,  0.0464],\n",
      "        ...,\n",
      "        [ 0.0780,  0.1085, -0.1384,  ...,  0.0078,  0.0049,  0.0406],\n",
      "        [-0.0077, -0.1199, -0.0079,  ...,  0.0218, -0.0002, -0.0411],\n",
      "        [ 0.1986,  0.0561,  0.1707,  ..., -0.0448, -0.0587,  0.0247]])), ('bert.encoder.layer.1.attention.self.key.bias', tensor([-8.4246e-04,  2.6007e-02,  5.8053e-03,  8.3772e-03, -1.8889e-02,\n",
      "         5.7541e-04,  7.7806e-03, -3.8970e-03, -1.5031e-02,  8.3124e-03,\n",
      "        -2.2915e-03, -1.4244e-02,  3.0006e-02, -7.0124e-03, -5.2836e-03,\n",
      "        -2.7191e-04, -2.1405e-02,  2.5236e-02, -2.7963e-02, -9.4756e-03,\n",
      "        -1.2042e-02,  1.5021e-02,  6.2994e-03,  5.2229e-03,  2.7717e-03,\n",
      "         3.7480e-03, -4.9163e-03, -9.5914e-03, -2.0720e-02,  1.1727e-02,\n",
      "        -2.3287e-03,  4.0742e-03,  1.3049e-03,  4.1577e-05, -4.0484e-04,\n",
      "        -9.0806e-04, -7.6698e-04,  1.2125e-03,  1.8711e-03,  1.4375e-03,\n",
      "        -6.3509e-04,  3.7371e-03,  1.2496e-03, -6.0131e-04, -1.4882e-04,\n",
      "         1.2112e-03,  3.7102e-03,  1.3649e-03,  2.1598e-03, -2.2457e-03,\n",
      "        -1.8201e-04,  1.3554e-03,  1.2955e-03,  3.0020e-03,  1.4293e-03,\n",
      "        -2.5163e-04, -1.5587e-03,  6.6788e-04, -5.5681e-04, -2.2538e-03,\n",
      "         8.1038e-05, -2.0758e-03,  1.1554e-03,  1.7624e-03,  1.3778e-02,\n",
      "        -3.0670e-03,  4.6566e-03,  1.1478e-02, -2.9772e-03,  1.2799e-02,\n",
      "        -9.9199e-03,  9.2005e-03,  5.8287e-03,  1.5493e-02, -1.0768e-02,\n",
      "        -5.9505e-04,  4.9473e-03,  2.0184e-02, -5.6069e-03, -6.2894e-03,\n",
      "         6.5779e-03, -1.4417e-02, -4.8520e-03,  1.7556e-02, -9.0582e-04,\n",
      "         8.8432e-03,  1.7905e-02,  3.7137e-03, -4.0815e-03, -2.7869e-03,\n",
      "         5.5170e-03,  2.2614e-03, -1.3162e-03,  1.7501e-02,  4.0874e-03,\n",
      "         2.8028e-03,  3.6315e-03, -3.1157e-02, -1.7005e-02, -1.5502e-02,\n",
      "        -1.5493e-02,  7.2540e-04,  2.6189e-03, -5.0726e-03,  2.6279e-02,\n",
      "         6.2485e-03,  1.9693e-02,  2.4558e-03, -4.7679e-03,  6.4156e-03,\n",
      "        -1.7870e-02, -1.1230e-02,  2.8020e-02,  1.9357e-03,  3.1722e-02,\n",
      "        -1.4402e-02,  2.1902e-02, -1.0627e-02, -1.8392e-02, -9.1824e-03,\n",
      "        -5.2510e-03,  1.5322e-02, -9.1742e-03, -6.2955e-03,  4.4229e-03,\n",
      "        -2.0925e-02,  1.4499e-02,  6.7966e-03,  8.7487e-03,  1.6836e-04,\n",
      "        -5.0865e-03,  1.5034e-02,  4.6736e-03, -1.2284e-02,  4.3564e-03,\n",
      "        -4.9510e-03, -1.0255e-02,  1.2245e-02, -9.0420e-03,  1.3793e-02,\n",
      "        -1.0867e-02,  6.4706e-03,  1.2317e-02,  5.0605e-03, -3.2026e-03,\n",
      "        -9.0821e-03, -1.8840e-02, -8.8835e-03,  1.2767e-02, -9.0895e-03,\n",
      "        -6.3074e-03,  7.9550e-03,  6.8316e-03,  6.5640e-03, -1.1292e-02,\n",
      "        -1.7428e-02, -4.1440e-03, -1.4413e-02, -5.0513e-03, -6.9048e-03,\n",
      "         1.3162e-02,  1.3540e-02,  7.3078e-03, -1.5077e-02, -3.0930e-03,\n",
      "         1.2456e-02,  6.5122e-03, -1.0353e-03,  1.0030e-02,  4.7306e-03,\n",
      "         1.2235e-02, -7.9818e-03, -1.3224e-02,  4.5126e-03,  2.5581e-03,\n",
      "        -4.7625e-03,  1.1125e-02, -7.2642e-03, -1.6658e-02, -6.4881e-03,\n",
      "        -4.0653e-03, -3.4790e-03, -2.2273e-03,  3.2117e-03, -4.6625e-03,\n",
      "         1.7027e-02,  5.3940e-03,  5.7368e-03, -2.0724e-03, -5.2803e-04,\n",
      "        -3.0749e-03,  6.5244e-03,  7.5196e-04, -1.0436e-03, -4.4441e-03,\n",
      "        -1.8549e-03,  7.8311e-03,  2.2061e-03,  3.6293e-03, -1.3351e-03,\n",
      "        -1.3236e-03,  1.2856e-03, -5.4151e-05, -4.3067e-03, -2.6196e-03,\n",
      "        -3.8365e-03, -4.4465e-03,  7.2296e-03, -4.1785e-03,  1.2787e-03,\n",
      "        -6.2103e-04,  3.0362e-03,  4.6832e-03,  1.3625e-03,  5.5445e-03,\n",
      "        -7.8852e-04, -7.0051e-03,  1.4557e-03, -6.9132e-05,  6.9390e-04,\n",
      "        -7.7714e-04,  4.4198e-03, -3.0266e-03, -7.1375e-04, -8.5336e-04,\n",
      "        -1.1350e-03,  1.2152e-03,  1.2475e-03,  1.1523e-03,  1.2096e-03,\n",
      "         2.0399e-03,  1.4803e-04, -7.4654e-05,  1.3654e-03, -1.1554e-03,\n",
      "        -5.6092e-04,  1.8079e-03, -2.1182e-04, -1.4606e-03,  3.5093e-03,\n",
      "         2.8026e-03, -2.8911e-03, -9.2481e-04,  2.0233e-03,  1.2058e-03,\n",
      "        -2.8999e-05, -3.4730e-03,  1.7341e-03, -3.3154e-03, -1.0548e-03,\n",
      "        -4.3106e-04, -3.3468e-04, -1.5524e-03,  1.4100e-03,  3.1067e-04,\n",
      "        -1.6070e-03,  1.0158e-02, -9.6346e-03,  9.4160e-03,  2.3841e-02,\n",
      "        -1.3176e-02,  1.2881e-02, -1.4564e-02,  1.2135e-02,  3.6312e-03,\n",
      "        -1.0565e-02, -1.4929e-02,  5.8659e-03,  7.9495e-03,  2.0812e-02,\n",
      "        -5.1362e-03, -6.0177e-03, -2.8869e-03, -8.5053e-04,  5.3751e-03,\n",
      "        -1.7778e-02,  5.2034e-05,  5.2782e-03, -4.4791e-03, -4.7937e-03,\n",
      "        -1.5566e-02,  7.7845e-03,  1.0881e-02,  5.7592e-03, -1.3035e-03,\n",
      "        -3.5274e-05,  4.9355e-03, -1.2947e-02,  2.3527e-03,  3.5143e-03,\n",
      "         2.3244e-03, -1.8494e-03,  5.6118e-04, -4.2323e-04,  2.3602e-03,\n",
      "         3.9159e-03, -1.6607e-03, -4.3279e-04,  1.9539e-03,  3.9359e-04,\n",
      "        -1.1497e-03, -3.9670e-04, -6.1607e-04,  2.0093e-04,  1.0924e-03,\n",
      "        -1.3089e-04,  3.0569e-03, -8.8376e-04, -5.4634e-04, -9.8818e-04,\n",
      "         8.7897e-04,  1.5161e-03,  3.0856e-05, -7.2812e-04,  1.7615e-03,\n",
      "         1.6027e-04, -6.6687e-05,  2.4355e-03, -1.1070e-03,  1.1880e-04,\n",
      "         2.9211e-03,  2.6268e-03,  1.3557e-02,  1.0360e-02,  1.2311e-02,\n",
      "        -4.3282e-03,  2.2910e-04, -1.0160e-03,  1.0763e-02,  3.6682e-03,\n",
      "         8.3731e-03,  1.3046e-02,  3.8527e-04, -9.5319e-03,  3.0020e-03,\n",
      "         3.4662e-03, -7.3960e-04, -1.9419e-02,  8.5162e-03, -7.7674e-03,\n",
      "         2.2504e-03,  1.1461e-02,  9.7170e-03,  1.4456e-02,  6.5592e-03,\n",
      "         2.3659e-03, -3.4038e-03, -5.9227e-03,  6.1832e-03,  5.3778e-03,\n",
      "         4.7448e-03, -4.4028e-03, -1.4879e-02,  6.6001e-03, -6.2092e-03,\n",
      "        -5.4045e-03, -1.8186e-03, -1.4836e-03,  1.2534e-03, -6.9626e-03,\n",
      "         6.9107e-03, -3.5926e-03,  2.6581e-03,  1.8490e-03,  9.3313e-03,\n",
      "         4.0016e-03, -1.4972e-03, -6.8414e-03, -3.5268e-04, -9.4958e-03,\n",
      "        -5.1070e-04, -4.3658e-03,  1.2541e-02,  5.0891e-03,  1.7358e-02,\n",
      "        -4.9174e-03, -6.9782e-03,  1.3913e-02, -7.1097e-03,  7.7145e-03,\n",
      "         4.7172e-03, -2.0049e-03,  3.9078e-03, -6.7962e-04])), ('bert.encoder.layer.1.attention.self.value.weight', tensor([[-0.0167, -0.0066, -0.0064,  ...,  0.0140, -0.0252, -0.0525],\n",
      "        [ 0.0119,  0.0041, -0.0028,  ...,  0.0120, -0.0021,  0.0502],\n",
      "        [ 0.0091,  0.0127,  0.0073,  ...,  0.0234,  0.0186,  0.0624],\n",
      "        ...,\n",
      "        [ 0.0133, -0.0066,  0.0113,  ..., -0.0476, -0.0585,  0.0246],\n",
      "        [-0.0165,  0.0147,  0.0046,  ...,  0.0520,  0.0558,  0.0011],\n",
      "        [-0.0132,  0.0018, -0.0020,  ...,  0.0061, -0.0260,  0.0091]])), ('bert.encoder.layer.1.attention.self.value.bias', tensor([-2.1499e-02, -1.4440e-02,  7.6921e-02,  4.0036e-02,  3.5657e-02,\n",
      "         6.7578e-02, -7.3030e-02,  6.0157e-02,  5.4890e-02, -2.4585e-02,\n",
      "         3.3208e-02, -2.0900e-02,  2.6385e-02, -3.1345e-02, -5.3930e-02,\n",
      "        -1.8774e-02,  3.3145e-02,  3.9839e-03,  3.3627e-03,  1.1668e-01,\n",
      "        -4.5736e-03,  4.8848e-02,  6.0633e-02,  1.2330e-02, -9.5276e-03,\n",
      "        -7.5712e-03, -6.9264e-02,  8.8721e-02, -1.7256e-02,  2.3947e-02,\n",
      "         1.5396e-02, -1.9581e-02,  2.5016e-03, -2.1166e-02, -2.4521e-02,\n",
      "         8.4912e-03, -9.9437e-03, -9.5269e-04,  6.2211e-03, -5.7880e-04,\n",
      "         2.1771e-03,  1.6183e-02, -4.1339e-02, -2.2833e-03,  5.2559e-03,\n",
      "         4.5912e-04, -7.1284e-03,  1.8952e-03, -1.0496e-03,  5.1964e-03,\n",
      "         7.9711e-03, -6.3195e-03,  1.5456e-02, -9.1729e-03,  1.0802e-02,\n",
      "        -4.6861e-02, -1.3747e-02,  1.5738e-02,  1.8064e-02, -3.2718e-02,\n",
      "        -2.9940e-02,  3.3782e-03,  7.1232e-04, -2.5028e-02, -1.2067e-02,\n",
      "         4.9066e-03, -7.3769e-03, -1.1925e-02, -1.4827e-02,  4.5602e-03,\n",
      "        -2.9761e-02, -5.1324e-03, -2.5846e-02, -7.9898e-03,  2.2824e-02,\n",
      "        -1.7569e-02,  4.6468e-04, -8.0537e-03, -1.7141e-02, -1.6475e-04,\n",
      "         2.2882e-02,  1.4752e-02,  1.5414e-02, -3.2835e-02,  1.6033e-02,\n",
      "         4.8634e-03,  1.1807e-03,  3.4452e-02, -9.3606e-03,  3.8849e-03,\n",
      "        -1.0019e-02, -4.1644e-03, -1.2367e-02,  1.6096e-02,  9.0141e-03,\n",
      "        -4.5233e-02, -7.6788e-03, -5.3808e-02,  5.5540e-02,  2.7219e-02,\n",
      "        -4.9651e-02,  3.1390e-02, -7.5592e-02, -4.1920e-02,  6.4030e-03,\n",
      "        -4.7058e-02, -6.7168e-02, -5.1091e-02,  5.6014e-02, -1.8570e-02,\n",
      "        -1.0691e-01,  1.6196e-02,  8.2515e-02, -6.2164e-02,  3.8648e-02,\n",
      "        -3.5161e-03,  1.8147e-02, -1.0636e-02,  1.6330e-02,  4.2065e-02,\n",
      "        -1.5699e-02,  4.5332e-02, -4.1648e-02, -1.4108e-02, -1.0133e-01,\n",
      "         6.0081e-02,  6.9458e-02,  7.4598e-02,  2.7883e-02,  1.9721e-02,\n",
      "         5.3095e-02, -2.5648e-02,  7.0526e-02,  4.5273e-02, -3.5883e-02,\n",
      "        -4.1054e-02,  3.2069e-02,  6.7168e-03,  1.5686e-02, -6.2377e-02,\n",
      "        -2.4761e-02,  9.8690e-02, -7.5071e-02,  5.4890e-02, -4.1309e-02,\n",
      "        -1.9817e-02, -6.2422e-02, -9.7847e-02,  3.9950e-03,  4.7830e-02,\n",
      "        -2.1403e-02,  4.3509e-02, -1.1721e-01,  6.0125e-02,  6.0988e-03,\n",
      "         1.1310e-01, -2.3385e-02,  7.6267e-03,  1.2776e-02, -4.0837e-02,\n",
      "         3.1413e-02,  2.9165e-03, -2.9652e-02,  1.7345e-02,  5.5492e-03,\n",
      "         5.6972e-03,  1.9313e-02, -6.7235e-03, -4.7885e-02,  7.9414e-05,\n",
      "        -5.0713e-03,  1.8418e-02,  5.1747e-02,  1.1517e-02, -4.8462e-03,\n",
      "         7.2811e-03, -1.9901e-03, -1.3183e-02,  5.7490e-03,  1.0892e-02,\n",
      "         7.2767e-03,  1.3111e-02, -1.5909e-03,  2.6314e-03, -3.0758e-02,\n",
      "         8.9224e-03, -2.1790e-02,  2.5720e-02,  1.4818e-02,  2.0294e-02,\n",
      "         1.6700e-02,  3.2645e-02,  2.0607e-02,  1.4734e-03,  5.6344e-03,\n",
      "        -4.0737e-02, -2.4154e-02,  1.5193e-02,  6.8569e-03, -2.0016e-02,\n",
      "        -1.1775e-02,  2.5686e-02,  2.7606e-03, -6.2812e-03, -7.9364e-03,\n",
      "         2.6187e-02,  1.6480e-02,  1.0112e-04, -3.2369e-03, -1.7481e-03,\n",
      "        -8.0225e-03,  9.1896e-03,  8.8764e-03,  6.4917e-03, -5.2919e-03,\n",
      "        -1.8020e-02,  2.1269e-02, -2.6551e-02,  1.1390e-02, -2.9726e-03,\n",
      "         7.2068e-03, -2.8805e-02, -3.0748e-03, -1.8598e-02, -2.2339e-03,\n",
      "        -4.7739e-03,  1.7303e-02,  2.6702e-02,  9.8598e-03,  8.7106e-03,\n",
      "        -5.9233e-03, -1.9158e-03,  4.9495e-03,  4.6241e-03,  3.6707e-03,\n",
      "         3.2113e-03, -3.9711e-03, -2.9687e-03,  1.0126e-02,  2.2363e-03,\n",
      "        -7.4407e-03, -3.1308e-02,  2.3382e-03, -6.4324e-04, -7.5767e-03,\n",
      "         2.4943e-02,  1.2052e-02,  3.2112e-03,  4.3752e-03,  2.5078e-02,\n",
      "        -1.9625e-02, -8.4242e-03, -3.3047e-02,  1.2505e-02,  1.3556e-03,\n",
      "         7.0962e-03, -6.3607e-02,  8.4516e-02,  1.8644e-02, -8.8419e-02,\n",
      "         5.7650e-02, -5.9462e-02,  1.1681e-02, -6.8537e-02,  6.0214e-03,\n",
      "         4.7416e-02, -4.0807e-02, -6.6268e-02, -3.1131e-02,  5.3381e-02,\n",
      "         4.0963e-02,  1.5968e-02, -1.8630e-02,  6.0705e-02, -3.6458e-02,\n",
      "         4.0792e-02,  6.2987e-02, -8.4321e-04, -7.1856e-02,  9.2816e-03,\n",
      "        -5.5385e-03, -3.7673e-02,  6.9655e-02,  7.0112e-03,  2.4196e-02,\n",
      "         5.7728e-02, -6.9086e-02, -5.9696e-02, -4.5923e-04,  1.1979e-02,\n",
      "        -4.5429e-03, -2.3269e-02,  1.1214e-02,  1.9517e-03,  2.3501e-02,\n",
      "         3.0555e-03, -1.0196e-02, -6.2899e-03,  3.1694e-03, -1.2398e-02,\n",
      "        -1.3187e-03, -3.5338e-03, -1.3703e-02, -4.0115e-03, -1.9719e-02,\n",
      "        -1.7863e-02, -7.7255e-03,  2.7087e-02, -2.9571e-03,  4.7725e-03,\n",
      "         1.4453e-03,  2.4648e-03, -4.7732e-03, -2.8766e-03, -7.3222e-03,\n",
      "        -1.2771e-02, -1.4483e-02, -2.7124e-02, -3.4698e-04,  1.0397e-02,\n",
      "         2.1607e-02,  8.3086e-04,  1.2309e-02, -4.5322e-02,  5.5875e-03,\n",
      "        -2.4260e-03, -6.9317e-03,  4.5001e-03,  7.2919e-03, -2.4707e-02,\n",
      "        -1.7472e-04, -1.7457e-03,  2.7356e-02,  6.2138e-02, -2.2126e-02,\n",
      "         4.4882e-02,  2.5233e-03,  1.9237e-02, -6.4518e-03, -5.9095e-02,\n",
      "        -2.5666e-05, -7.2062e-03,  1.4968e-02,  1.1773e-02,  8.8329e-03,\n",
      "         2.0870e-02,  6.2060e-03, -3.0660e-02,  3.4470e-03, -4.6912e-02,\n",
      "         3.3089e-02, -2.1385e-02,  3.4191e-02,  6.0390e-02,  4.1965e-02,\n",
      "        -1.4484e-02,  2.4223e-02, -4.4923e-02, -6.6531e-03, -3.0401e-02,\n",
      "        -2.7566e-03,  4.9479e-03, -1.1011e-03, -4.3498e-03,  2.6582e-02,\n",
      "         2.0982e-02, -1.5697e-02,  1.9279e-02,  5.9416e-02, -4.4617e-02,\n",
      "         2.1043e-02, -1.4205e-02, -1.6375e-02,  1.0931e-02, -1.1234e-02,\n",
      "        -1.4452e-02,  9.7890e-03,  9.0654e-03,  1.6343e-02,  2.2206e-02,\n",
      "         1.9440e-02, -2.1197e-02, -3.4471e-03, -1.6580e-02])), ('bert.encoder.layer.1.attention.output.dense.weight', tensor([[-0.0195,  0.1295, -0.1833,  ...,  0.0515,  0.0221, -0.0998],\n",
      "        [ 0.0988, -0.0865, -0.0600,  ...,  0.0459,  0.0606,  0.0369],\n",
      "        [ 0.0078,  0.0283,  0.0980,  ..., -0.0706, -0.1889, -0.0859],\n",
      "        ...,\n",
      "        [ 0.0125, -0.0080,  0.0301,  ...,  0.0285,  0.0186, -0.0061],\n",
      "        [-0.0652, -0.0470,  0.0094,  ..., -0.0555,  0.0343, -0.0265],\n",
      "        [-0.0447,  0.0101, -0.0284,  ...,  0.0102,  0.0131,  0.0185]])), ('bert.encoder.layer.1.attention.output.dense.bias', tensor([-0.0486, -0.1259, -0.0616, -0.0510, -0.0595, -0.1353, -0.0738,  0.0185,\n",
      "        -0.0260, -0.1299,  0.1469, -0.0067,  0.0924,  0.0469,  0.1023, -0.0443,\n",
      "        -0.0380, -0.0010, -0.0133, -0.0235,  0.0055, -0.0262, -0.0736,  0.0420,\n",
      "         0.0529,  0.0258, -0.0382, -0.0714, -0.0999, -0.1029, -0.0386,  0.0106,\n",
      "        -0.0659, -0.0419, -0.0022,  0.0025, -0.0073, -0.0607,  0.0305,  0.0054,\n",
      "        -0.0653, -0.1600, -0.0414,  0.0213, -0.1963,  0.0899, -0.0383,  0.0209,\n",
      "        -0.0836, -0.0273,  0.0318, -0.0953,  0.0515, -0.0138, -0.0531,  0.0053,\n",
      "        -0.1224, -0.0169, -0.1522,  0.0447,  0.0786, -0.0251, -0.0491, -0.0062,\n",
      "        -0.0543,  0.0086, -0.0257,  0.0190,  0.0369,  0.0388, -0.0326, -0.0466,\n",
      "         0.0126,  0.0220, -0.0159, -0.0205, -0.0202, -0.0485, -0.0095,  0.0119,\n",
      "         0.0307,  0.1565, -0.0040, -0.0413,  0.0323, -0.0394,  0.0870,  0.0385,\n",
      "        -0.0602,  0.0140, -0.0119,  0.0645,  0.0158, -0.0130, -0.0668, -0.0646,\n",
      "         0.0702, -0.0366,  0.0190, -0.0545,  0.0888, -0.0489, -0.0266, -0.0334,\n",
      "         0.0377,  0.1682,  0.1075, -0.0277,  0.0267,  0.0011, -0.0118, -0.0169,\n",
      "        -0.0354,  0.0282,  0.0693,  0.0121,  0.0072,  0.0089,  0.0801, -0.0204,\n",
      "        -0.0109,  0.0249,  0.0884,  0.0155, -0.0681,  0.0100,  0.0485, -0.0288,\n",
      "        -0.0595, -0.0764,  0.0867,  0.0181,  0.0461,  0.1083,  0.0941, -0.0018,\n",
      "         0.0359, -0.0426,  0.0778,  0.0278,  0.0189, -0.0282,  0.0431,  0.0098,\n",
      "        -0.0576, -0.0301, -0.0106,  0.0394, -0.0651,  0.0247, -0.0094,  0.0507,\n",
      "         0.0227, -0.0085, -0.1218, -0.0468, -0.0638,  0.0435, -0.0069,  0.2315,\n",
      "        -0.0773, -0.0268,  0.0487, -0.0224, -0.0849, -0.0456, -0.0765,  0.0601,\n",
      "        -0.1141, -0.0099,  0.0285, -0.0184, -0.0107,  0.0283,  0.0472, -0.0746,\n",
      "        -0.0219, -0.0093,  0.0409,  0.0363, -0.0185,  0.1988, -0.1128,  0.0291,\n",
      "         0.0403,  0.0040,  0.0304,  0.2367, -0.0073,  0.0566, -0.0123,  0.0034,\n",
      "        -0.0291, -0.0561, -0.0194, -0.0336,  0.0296,  0.0343,  0.0233,  0.0035,\n",
      "         0.1488,  0.0451,  0.0387,  0.0087, -0.0949,  0.0336,  0.0006, -0.0314,\n",
      "        -0.0292, -0.0190,  0.0147, -0.0666, -0.0086, -0.0414,  0.0085,  0.0562,\n",
      "         0.0324,  0.0056, -0.0168, -0.0327,  0.0757, -0.0164, -0.3210,  0.0097,\n",
      "        -0.0703,  0.0274,  0.0444, -0.0007, -0.0107,  0.0040, -0.0403, -0.0198,\n",
      "         0.0429,  0.0766, -0.0221, -0.0595, -0.2205, -0.0380,  0.0127,  0.0686,\n",
      "        -0.0400, -0.0982,  0.0405,  0.0750,  0.0890,  0.0528,  0.0031, -0.0309,\n",
      "         0.0601, -0.0428, -0.0332,  0.0769,  0.0040, -0.0239,  0.0477,  0.0478,\n",
      "         0.0035,  0.0564,  0.0558, -0.0663, -0.0241,  0.0398, -0.0320,  0.0111,\n",
      "        -0.0290,  0.0942,  0.0542,  0.0113, -0.0211,  0.0337, -0.0444,  0.0318,\n",
      "        -0.0006, -0.0219, -0.0828,  0.0897, -0.0136, -0.0088, -0.0394,  0.0736,\n",
      "        -0.0044, -0.0136,  0.0058,  0.0345, -0.0223, -0.0245, -0.0420, -0.0038,\n",
      "         0.0082,  0.0522, -0.0895,  0.0788,  0.0438,  0.0013, -0.0161, -0.0162,\n",
      "        -0.0502, -0.0532,  0.0363,  0.0146, -0.0677,  0.0017,  0.0135,  0.0202,\n",
      "         0.0454, -0.0022, -0.0646, -0.0571, -0.0212,  0.0303,  0.0360, -0.0740,\n",
      "         0.0310, -0.0430, -0.0867,  0.1065, -0.0730, -0.0146,  0.0019,  0.0540,\n",
      "         0.0072,  0.0463,  0.0237, -0.0275, -0.0297, -0.0340,  0.0349,  0.0400,\n",
      "        -0.0298,  0.1116,  0.0022, -0.0907,  0.0233, -0.1303, -0.0708, -0.0208,\n",
      "        -0.0202, -0.0387, -0.0299, -0.0014,  0.0309,  0.0148, -0.0429, -0.0386,\n",
      "        -0.0246, -0.0136, -0.0061,  0.0854, -0.0939,  0.0250, -0.0364, -0.0634,\n",
      "         0.0945, -0.0132, -0.0732,  0.0663, -0.0951,  0.0706, -0.0500,  0.0370,\n",
      "        -0.0180,  0.0491,  0.0264, -0.0998, -0.0328,  0.0095,  0.0069,  0.0497,\n",
      "         0.0326,  0.0173,  0.0060,  0.0740, -0.0390,  0.0497,  0.0879,  0.0443,\n",
      "         0.0419, -0.0041, -0.0437, -0.0830, -0.1065, -0.0648,  0.0219,  0.0335])), ('bert.encoder.layer.1.attention.output.LayerNorm.weight', tensor([0.4434, 0.3842, 0.3453, 0.4415, 0.2943, 0.3692, 0.3205, 0.3791, 0.2031,\n",
      "        0.4511, 0.4840, 0.5984, 0.5354, 0.4862, 0.4318, 0.4282, 0.5053, 0.4441,\n",
      "        0.4424, 0.4571, 0.5910, 0.5311, 0.6334, 0.7929, 0.9963, 0.8803, 0.8937,\n",
      "        0.7801, 0.7181, 0.6394, 0.7008, 0.5421, 0.6616, 0.6510, 0.6968, 0.8029,\n",
      "        0.9040, 0.8800, 1.0165, 0.9473, 0.8983, 0.6990, 0.8421, 0.8819, 0.6748,\n",
      "        0.6556, 0.8075, 0.7075, 0.8117, 0.9427, 0.9649, 0.9615, 1.0752, 0.9626,\n",
      "        0.9804, 0.9707, 0.9992, 0.8418, 0.9137, 0.8905, 0.9114, 0.8831, 0.9100,\n",
      "        0.9110, 0.9398, 1.0044, 1.0547, 0.9958, 0.9899, 0.9813, 1.0139, 1.0211,\n",
      "        0.9685, 1.0066, 0.9828, 1.0975, 0.9944, 1.0718, 0.9901, 1.1031, 1.0821,\n",
      "        0.8794, 1.0328, 1.0074, 1.0961, 0.9582, 1.0326, 1.0782, 1.0203, 1.0049,\n",
      "        1.0634, 1.0748, 1.0736, 0.9316, 1.0968, 0.9804, 1.0353, 1.0255, 1.0516,\n",
      "        1.1199, 1.0858, 0.7393, 1.0807, 1.1060, 1.0958, 0.6161, 1.0834, 0.9744,\n",
      "        1.1313, 1.0256, 1.0413, 1.0916, 1.1278, 1.1067, 1.0977, 1.0978, 1.0397,\n",
      "        1.1354, 1.0913, 0.8300, 1.3313, 1.1539, 1.0694, 1.1036, 1.1379, 1.1135,\n",
      "        1.1290, 0.9352, 1.0239, 1.0821, 0.9482, 1.1215, 1.0262, 1.1232, 1.1217,\n",
      "        1.0948, 1.0819, 1.1018, 1.0752, 1.0739, 1.0976, 1.0805, 1.1527, 1.0978,\n",
      "        1.1129, 1.1032, 1.0763, 1.0170, 1.0433, 1.0875, 1.1242, 1.1015, 1.1245,\n",
      "        1.0761, 0.8571, 1.0730, 0.9519, 0.9325, 1.0488, 1.0002, 1.1315, 1.1327,\n",
      "        1.0754, 1.1351, 1.0833, 1.0381, 1.1412, 1.0363, 0.9115, 1.1290, 1.0548,\n",
      "        1.0517, 1.0769, 1.1032, 1.1448, 1.0586, 1.1266, 1.0877, 1.0988, 0.9657,\n",
      "        1.0245, 0.8618, 1.1016, 1.0441, 1.0101, 1.1096, 1.1372, 1.5016, 1.1102,\n",
      "        1.0787, 1.0522, 1.1096, 1.0558, 1.0237, 1.0530, 0.9825, 1.1225, 1.0921,\n",
      "        1.1222, 1.0094, 1.1118, 1.1071, 1.0242, 1.0762, 1.1029, 1.0444, 0.9101,\n",
      "        1.0641, 1.1466, 1.0460, 1.0414, 1.1195, 1.1220, 1.0765, 1.0051, 1.1418,\n",
      "        1.0865, 1.0488, 0.8230, 1.0432, 1.1274, 1.0675, 0.8106, 1.0331, 1.0387,\n",
      "        1.0996, 1.0446, 1.1863, 1.1153, 1.0708, 1.1583, 1.1063, 1.0995, 1.0386,\n",
      "        1.0723, 1.1020, 0.8168, 1.1630, 1.0104, 1.0833, 1.0316, 1.0962, 1.1089,\n",
      "        1.0980, 1.1291, 1.1106, 1.0555, 0.9122, 1.2005, 1.0952, 1.0888, 1.0545,\n",
      "        1.1513, 1.1619, 1.0936, 1.0938, 1.1303, 1.0992, 0.8971, 1.0893, 1.0816,\n",
      "        1.1163, 1.1396, 1.1138, 1.0647, 1.1098, 1.1025, 1.1211, 1.0403, 1.0637,\n",
      "        1.1467, 0.9961, 1.1049, 1.0818, 1.0271, 1.1064, 1.1456, 1.0246, 1.1270,\n",
      "        0.9688, 1.1260, 1.1493, 1.1501, 0.9577, 1.1303, 1.1081, 1.1252, 0.6716,\n",
      "        1.0728, 1.1154, 1.1008, 1.0889, 1.0578, 1.1197, 1.0207, 1.1057, 1.1844,\n",
      "        1.1087, 1.1227, 1.0296, 1.0991, 1.0046, 1.1331, 1.1071, 1.0981, 1.0970,\n",
      "        1.0750, 1.0578, 1.1034, 1.0136, 1.0395, 1.0211, 1.1156, 1.0826, 1.0520,\n",
      "        1.0653, 1.0652, 1.1263, 1.0801, 1.1183, 1.0734, 1.0861, 1.1237, 1.0844,\n",
      "        1.1049, 1.1710, 1.0198, 1.1012, 1.1186, 0.9769, 1.0954, 1.0462, 1.1394,\n",
      "        0.7688, 0.8753, 1.1515, 1.0860, 1.0050, 1.1569, 1.1155, 1.1555, 1.0579,\n",
      "        1.0829, 1.1010, 1.0295, 1.1413, 1.0873, 1.0662, 1.0945, 1.1371, 1.0488,\n",
      "        1.0560, 1.0372, 0.9726, 1.0682, 1.1781, 1.1353, 0.9817, 1.0781, 1.0754,\n",
      "        1.0254, 1.0207, 1.1111, 0.6464, 1.0433, 1.0985, 1.0960, 1.0282, 1.0480,\n",
      "        0.8302, 1.1152, 1.0455, 1.1240, 1.1084, 1.0879, 1.0322, 1.0211, 1.0188,\n",
      "        1.0839, 1.0827, 1.0440, 1.0813, 1.1293, 1.1359])), ('bert.encoder.layer.1.attention.output.LayerNorm.bias', tensor([-3.4263e-02, -9.8761e-02, -9.8937e-02,  4.7510e-02,  1.1968e-01,\n",
      "        -1.4223e-01, -7.2864e-02, -4.4559e-02, -2.6100e-02, -1.2074e-01,\n",
      "        -1.2493e-01, -7.1546e-03,  3.4470e-02, -2.1558e-01,  2.1959e-02,\n",
      "        -1.7660e-02, -1.5457e-01, -4.7124e-02, -5.1490e-02,  1.2831e-02,\n",
      "        -6.9692e-02, -9.9386e-02, -1.1784e-01,  2.0161e-01,  8.1267e-02,\n",
      "         3.9210e-02, -5.5716e-02,  4.6099e-02, -4.5773e-02, -2.1464e-01,\n",
      "        -7.8618e-02, -6.9690e-02,  1.1206e-02,  1.0147e-02, -3.3618e-02,\n",
      "        -8.7354e-02, -4.8474e-02, -8.9645e-02, -5.1343e-02,  2.8699e-02,\n",
      "        -6.7187e-02, -1.7204e-01, -1.2115e-01, -8.9982e-02,  3.5750e-02,\n",
      "        -1.8794e-01, -1.4195e-01,  7.7106e-02, -2.5014e-01,  7.0083e-02,\n",
      "         1.3143e-02,  1.5701e-02,  4.2361e-02,  9.7407e-02, -2.3469e-02,\n",
      "        -3.7762e-02, -2.4158e-01, -1.1073e-01, -1.2934e-01, -4.7561e-02,\n",
      "         3.2350e-02, -7.9887e-02,  1.3555e-03, -2.3560e-02, -1.5679e-01,\n",
      "        -1.3353e-01,  1.4763e-01,  3.0276e-03,  1.6002e-01,  5.4245e-02,\n",
      "         7.8624e-03, -1.5487e-04,  8.7578e-02, -1.0199e-02, -3.3314e-01,\n",
      "         2.0011e-02,  1.1308e-01, -7.8762e-02, -9.1918e-02, -4.8986e-02,\n",
      "         2.2649e-01,  3.7818e-01,  1.3428e-01, -2.4028e-02, -5.0441e-03,\n",
      "        -9.5321e-02,  8.7482e-02,  1.5605e-01, -1.1723e-01,  2.8085e-01,\n",
      "         2.6546e-02, -1.2897e-01,  2.8159e-02,  1.2855e-01,  1.0084e-01,\n",
      "         1.8846e-01,  4.0838e-02,  2.9816e-02, -8.0370e-02,  8.8332e-03,\n",
      "        -1.6307e-02,  2.0422e-01, -3.9522e-02, -8.2215e-03, -6.4336e-02,\n",
      "         1.5282e-01, -1.4852e-02,  1.4327e-01,  2.0118e-02,  1.3052e-01,\n",
      "         1.9656e-02,  9.7063e-02, -1.1073e-01,  1.7790e-01,  5.9311e-02,\n",
      "        -4.4990e-02,  5.5245e-02, -7.8673e-02,  3.2076e-02,  5.5756e-04,\n",
      "         2.4198e-02,  6.3693e-02,  5.4347e-02,  9.1340e-02, -7.7998e-03,\n",
      "        -1.7136e-01, -8.2666e-03,  1.1297e-01, -1.7652e-01,  1.0575e-01,\n",
      "         1.5492e-01,  1.1956e-01, -6.6005e-02, -9.2787e-03,  2.8788e-02,\n",
      "         7.7184e-02,  2.1846e-02, -3.7466e-02, -6.0379e-02, -6.7031e-02,\n",
      "        -1.2181e-02,  1.2446e-01,  2.1911e-02,  6.2312e-04, -3.6826e-02,\n",
      "        -3.3559e-02, -1.5368e-01, -1.7815e-02, -3.4990e-02,  1.0457e-01,\n",
      "        -3.8568e-03,  3.3482e-02, -1.7525e-02,  3.2992e-02,  3.4999e-02,\n",
      "         2.3474e-03, -1.1261e-01,  1.0801e-01, -6.5045e-02,  5.9134e-01,\n",
      "        -7.2283e-02,  1.0211e-02, -6.5829e-02,  1.4782e-01, -5.4497e-02,\n",
      "         3.2942e-01, -6.0876e-02,  2.1026e-01, -6.3654e-02,  2.2952e-02,\n",
      "         1.0201e-01, -1.0624e-01, -3.8744e-02,  1.9752e-01, -6.6057e-04,\n",
      "         9.3596e-02, -9.1899e-03,  1.0752e-02, -1.3069e-02,  1.6825e-01,\n",
      "        -2.3597e-02,  2.0267e-01,  3.0085e-02,  6.8909e-02, -1.4741e-01,\n",
      "        -1.2344e-01, -7.4665e-02,  8.8724e-01,  6.4185e-02,  2.0682e-01,\n",
      "        -1.1789e-02,  3.2779e-02, -3.0757e-02,  5.1247e-02, -2.5448e-01,\n",
      "         3.8893e-02,  6.9655e-02,  3.6485e-02,  4.8864e-02,  3.2005e-02,\n",
      "        -1.1409e-01,  4.8821e-02, -7.6378e-02,  3.1971e-02, -1.2173e-01,\n",
      "         4.5731e-02,  3.2164e-02,  1.8390e-01, -1.6278e-02,  1.2875e-01,\n",
      "        -3.0822e-02,  1.7284e-02, -5.7393e-02, -2.0390e-02, -6.0147e-02,\n",
      "         3.8487e-03, -6.2909e-02, -3.2754e-02, -1.4280e-01,  1.5669e-02,\n",
      "        -1.7205e-01, -9.4783e-02, -4.0087e-01, -4.9144e-02, -2.8842e-01,\n",
      "         4.2161e-02,  1.5565e-02,  1.1424e-01,  3.4913e-02, -1.0661e-02,\n",
      "        -1.1670e-01,  3.1128e-02, -1.6001e-01,  1.0999e-01, -1.7794e-03,\n",
      "         4.6778e-02, -1.0801e-01, -1.9797e-02, -4.8418e-02,  3.1180e-02,\n",
      "        -1.6248e-01,  7.5994e-02,  3.4365e-04,  9.6725e-02,  3.3667e-02,\n",
      "         3.2712e-01, -3.0978e-01,  4.4859e-02,  3.8137e-03,  5.3951e-04,\n",
      "        -4.4364e-02,  2.0248e-01, -1.5645e-01, -1.4722e-01,  2.1414e-02,\n",
      "         1.2012e-01,  4.3035e-02,  2.2287e-03, -2.5619e-01, -2.3086e-02,\n",
      "         1.0412e-02, -3.2961e-02, -7.1056e-02,  2.9103e-02,  3.5372e-03,\n",
      "         1.8457e-01,  5.8117e-02,  1.0168e-01, -1.3789e-01, -4.3008e-03,\n",
      "        -3.5780e-02,  2.6204e-01, -1.3446e-01, -6.4393e-03, -1.0959e-01,\n",
      "         1.9854e-01, -9.4965e-02,  1.1583e-01, -1.3169e-02,  2.2635e-01,\n",
      "        -8.9058e-02, -1.7832e-02, -5.7136e-02, -7.5889e-02, -8.3318e-02,\n",
      "         1.3447e-01, -8.9579e-02,  2.3464e-01, -5.3135e-02, -6.7623e-02,\n",
      "        -6.4598e-02,  1.4538e-01, -1.6125e-01, -2.0895e-02, -1.5415e-01,\n",
      "         1.0370e-01, -7.2738e-02,  3.6695e-02, -1.8804e-01,  1.5815e-01,\n",
      "        -1.6319e-01, -6.0873e-02,  8.9023e-03,  7.1940e-03, -3.1470e-02,\n",
      "         1.0983e-02, -1.6763e-01,  1.9983e-01, -7.3075e-02,  1.4690e-01,\n",
      "        -9.0865e-02, -3.4489e-02,  4.7093e-02, -4.2163e-02, -4.7822e-02,\n",
      "         5.0885e-02, -1.0664e-01,  4.2726e-02,  6.7695e-02,  1.6562e-02,\n",
      "         4.6804e-02, -1.8300e-02, -8.9620e-02,  1.4619e-01, -3.4399e-02,\n",
      "         1.2648e-01, -1.0752e-01, -3.5544e-02,  8.1539e-02,  1.6091e-01,\n",
      "        -4.1858e-02,  8.1992e-02, -1.0351e-01,  1.2777e-01, -6.4235e-02,\n",
      "         1.0215e-01,  5.7828e-02,  1.0493e-01,  1.7180e-02,  4.3090e-02,\n",
      "        -5.4138e-02,  4.3727e-02, -8.8898e-02,  8.7114e-02, -1.5692e-01,\n",
      "         2.6281e-02, -1.0478e-02,  6.0758e-02, -4.0248e-01,  2.2211e-01,\n",
      "        -1.7505e-01, -5.7461e-02, -8.9182e-02, -3.1674e-02, -2.7048e-02,\n",
      "         1.8275e-01, -8.8663e-02,  1.3475e-01,  4.4021e-04,  1.7290e-01,\n",
      "        -8.6512e-02,  1.7579e-02, -4.7900e-02, -6.7576e-02,  3.7585e-02,\n",
      "        -2.1730e-01,  9.5947e-03,  1.7735e-01, -1.0119e-01,  2.0814e-01,\n",
      "        -1.7209e-01, -4.5894e-02, -5.4724e-02, -5.8778e-02,  3.0008e-03,\n",
      "         6.9807e-03,  8.0158e-02,  8.8525e-02, -7.1976e-02,  1.2121e-01,\n",
      "        -8.5055e-03, -6.2689e-02, -1.8109e-01,  1.0465e-01])), ('bert.encoder.layer.1.intermediate.dense.weight', tensor([[ 0.0095,  0.0144, -0.0174,  ...,  0.0384, -0.0152,  0.0417],\n",
      "        [-0.0423, -0.0719, -0.0432,  ..., -0.0585,  0.1079, -0.0179],\n",
      "        [-0.1328,  0.0600,  0.0791,  ...,  0.1083,  0.0700,  0.0091],\n",
      "        ...,\n",
      "        [ 0.0224,  0.0333,  0.0626,  ..., -0.0095,  0.0415, -0.0297],\n",
      "        [-0.0710, -0.0171,  0.0738,  ...,  0.0168, -0.0427,  0.0732],\n",
      "        [-0.0057,  0.0163, -0.0323,  ..., -0.0418,  0.0482,  0.0080]])), ('bert.encoder.layer.1.intermediate.dense.bias', tensor([-0.0634, -0.0551, -0.0858,  ..., -0.0797, -0.0678, -0.0757])), ('bert.encoder.layer.1.output.dense.weight', tensor([[ 0.0810, -0.0616,  0.0356,  ..., -0.0388,  0.0017, -0.0145],\n",
      "        [-0.0372,  0.1000, -0.0202,  ...,  0.0133,  0.0400, -0.0710],\n",
      "        [-0.0565, -0.0969, -0.0690,  ..., -0.0555,  0.0181,  0.1385],\n",
      "        ...,\n",
      "        [ 0.0013, -0.0290, -0.0299,  ..., -0.0947,  0.0429,  0.0435],\n",
      "        [-0.0446, -0.0629, -0.0312,  ..., -0.0822, -0.0854, -0.0010],\n",
      "        [-0.0520, -0.0076, -0.0513,  ...,  0.0670,  0.0807, -0.0046]])), ('bert.encoder.layer.1.output.dense.bias', tensor([ 3.8205e-02, -2.2085e-02, -1.7602e-01, -1.5887e-02, -2.8484e-02,\n",
      "         1.1721e-01,  1.6447e-01, -2.9735e-02,  1.3922e-02,  1.2148e-01,\n",
      "         1.5860e-02,  3.0304e-02, -4.7690e-02, -1.4837e-02,  6.5256e-02,\n",
      "         3.4802e-02, -1.3225e-01, -1.0395e-02,  4.7953e-02, -1.0791e-01,\n",
      "        -2.6269e-02,  7.1289e-02,  4.8351e-02, -1.2761e-01, -8.7425e-03,\n",
      "        -3.0648e-02, -4.8872e-03, -1.6137e-02, -1.4469e-01, -1.8419e-03,\n",
      "        -5.0000e-02, -1.3075e-01,  1.1750e-02, -1.2706e-01, -1.0620e-01,\n",
      "        -7.0583e-02, -8.0311e-02,  1.5090e-01, -6.7688e-02,  1.2948e-03,\n",
      "         2.4591e-02, -6.0385e-02, -6.0832e-02, -2.2741e-02, -1.2842e-01,\n",
      "        -4.9892e-02, -1.9614e-02, -3.6832e-02,  6.0583e-02,  6.6378e-02,\n",
      "         1.0494e-01, -2.0212e-02,  4.1248e-02,  9.8738e-02, -7.0903e-02,\n",
      "         6.6586e-02, -2.1372e-02, -1.1239e-01,  1.0488e-01,  2.1026e-02,\n",
      "         9.1280e-02,  3.2633e-02, -1.1254e-01, -2.0173e-02,  1.1576e-01,\n",
      "         4.7040e-02,  8.4568e-02, -3.0074e-02,  4.8573e-02, -8.3307e-02,\n",
      "         7.4486e-02,  1.2046e-01,  8.6500e-02,  8.9502e-02,  6.8166e-02,\n",
      "         2.4778e-02,  6.2484e-02, -1.6973e-02,  1.7608e-02,  5.9037e-02,\n",
      "        -2.6540e-02, -6.9288e-02,  8.4744e-02,  2.8410e-02,  9.0880e-03,\n",
      "         2.0685e-02, -3.7924e-02,  8.6877e-02,  4.8929e-02,  1.3106e-01,\n",
      "        -8.6088e-03, -6.6017e-02,  4.0204e-03, -6.7387e-03,  2.4953e-02,\n",
      "         8.0089e-04, -2.7998e-02, -1.8728e-02, -3.0552e-02, -3.0415e-02,\n",
      "         4.9448e-02,  2.2458e-01,  4.1313e-03,  3.6737e-02,  1.4567e-02,\n",
      "        -8.3812e-02,  2.3133e-02,  4.4900e-02, -2.7168e-02,  7.9958e-02,\n",
      "        -5.4059e-02,  7.0659e-03, -6.1351e-02,  5.6456e-02,  7.4623e-02,\n",
      "         3.0005e-02,  6.4807e-02, -2.4372e-02, -6.0210e-03, -4.5648e-02,\n",
      "        -1.9643e-02,  1.4944e-02,  2.8934e-02,  8.1694e-02, -1.1988e-02,\n",
      "        -3.2332e-03,  4.4753e-03, -9.6176e-03,  4.0310e-02,  2.2772e-03,\n",
      "        -9.9342e-02,  5.0573e-02,  4.5440e-02,  5.7179e-03,  2.3214e-02,\n",
      "         2.3617e-02, -2.9880e-02, -1.9313e-02, -3.5207e-02,  1.9844e-02,\n",
      "        -1.5863e-02, -9.0235e-03,  1.8838e-02, -2.7979e-02, -4.6973e-03,\n",
      "         1.8204e-02, -4.3301e-02, -1.4709e-01, -8.5221e-02,  4.3730e-02,\n",
      "        -3.9737e-02,  2.8921e-02,  2.2686e-02, -3.2780e-02, -3.5371e-03,\n",
      "        -4.3805e-02,  6.7966e-03,  2.9172e-02, -5.9047e-02, -1.0598e-01,\n",
      "        -4.5464e-02, -3.1407e-02, -1.5080e-02,  4.2495e-02, -7.9362e-03,\n",
      "         1.5876e-02, -4.4362e-02, -3.1587e-02, -9.1940e-02,  3.3384e-02,\n",
      "         8.3534e-02, -1.8657e-03,  9.7793e-03,  2.1302e-02,  1.7505e-02,\n",
      "        -4.1245e-04, -4.1534e-02, -1.0653e-03,  6.6747e-02,  7.8295e-02,\n",
      "         3.5120e-02, -6.3633e-02, -1.6647e-02, -1.7558e-04, -4.1555e-02,\n",
      "        -1.0549e-02,  1.2566e-02, -3.8398e-02,  6.1295e-03,  6.0156e-02,\n",
      "        -5.9580e-02,  1.0554e-02, -8.7492e-03,  7.7023e-03, -1.1717e-01,\n",
      "        -5.8609e-02, -1.9557e-02,  2.6367e-03, -5.6444e-02, -2.9042e-02,\n",
      "         2.5943e-04,  1.3590e-02,  1.0316e-02,  4.8733e-02, -5.8564e-02,\n",
      "         2.7592e-02,  3.0779e-02,  3.9971e-02,  3.5141e-02,  5.3148e-02,\n",
      "        -4.7813e-02, -1.8141e-02,  5.6876e-02, -8.6826e-02, -3.6007e-02,\n",
      "         4.5132e-02, -6.2227e-02,  6.3831e-02,  3.3682e-02, -2.9649e-02,\n",
      "        -4.4744e-02, -8.3999e-03, -6.6670e-02,  1.5125e-02, -9.6707e-02,\n",
      "         9.0048e-03,  4.3990e-02,  3.5540e-02,  3.1249e-02, -1.2173e-02,\n",
      "         1.0574e-01, -4.2009e-02, -5.4594e-02,  1.6528e-02, -4.7935e-02,\n",
      "        -4.3744e-02, -1.0922e-01, -1.8970e-02, -9.6760e-03,  6.7777e-02,\n",
      "         8.8791e-03,  3.7303e-02, -3.4797e-02,  4.5264e-02,  4.0097e-02,\n",
      "         3.1902e-02, -1.0665e-02, -4.2289e-02,  6.0670e-02,  3.3441e-02,\n",
      "        -3.3164e-02, -5.3233e-03, -1.3227e-02, -3.8943e-02, -1.8825e-02,\n",
      "         2.6603e-02,  2.7784e-02, -3.5332e-02, -7.9527e-02, -5.4361e-02,\n",
      "         5.6774e-02, -4.0650e-02, -4.0193e-04,  2.8727e-02, -7.7204e-02,\n",
      "         4.3545e-02,  3.4625e-02,  4.7374e-03, -3.7222e-02,  6.2235e-02,\n",
      "        -6.1604e-02,  6.2694e-02,  2.2818e-02,  8.1115e-02, -3.4388e-02,\n",
      "         1.0932e-01,  7.0779e-03, -4.8232e-02, -1.3367e-02,  5.3290e-02,\n",
      "        -7.5526e-03, -8.1311e-03,  4.6848e-02, -4.7073e-02, -1.5857e-02,\n",
      "        -3.9544e-02, -5.0308e-03,  6.1143e-02, -2.2678e-02, -8.8037e-02,\n",
      "        -4.7003e-02,  1.8499e-02, -6.7880e-03, -2.7297e-03,  1.1493e-02,\n",
      "        -4.5561e-03, -1.7565e-02,  2.1880e-02, -5.2743e-02, -1.0679e-02,\n",
      "        -7.3991e-02, -4.2780e-02,  4.7462e-02,  2.6762e-02,  3.4052e-02,\n",
      "         6.5373e-02, -2.7379e-02,  6.3972e-02, -4.4056e-02, -6.1809e-02,\n",
      "         1.9639e-02,  4.0181e-02, -9.5197e-03,  4.6483e-02, -2.4961e-02,\n",
      "         2.4470e-02,  3.6409e-02,  7.9062e-02,  5.3824e-03,  3.4732e-03,\n",
      "         2.6125e-02,  1.0441e-02, -7.0793e-02,  4.8962e-02,  2.8506e-02,\n",
      "        -6.3378e-02, -3.2453e-02, -1.9168e-02,  6.2818e-02,  6.6066e-02,\n",
      "         1.0333e-02,  1.6761e-02,  1.3142e-02, -9.1161e-02,  2.9062e-02,\n",
      "         9.4677e-02,  6.6646e-02,  5.2965e-02,  9.0174e-03,  2.2131e-02,\n",
      "        -5.1303e-03, -5.9611e-02,  4.2505e-02,  2.7710e-02,  1.3507e-02,\n",
      "        -2.8889e-02,  3.0336e-02, -6.3411e-02, -1.0095e-01,  3.7781e-02,\n",
      "        -7.2136e-02,  3.4169e-02, -6.0135e-02, -4.7286e-02, -8.2805e-02,\n",
      "         5.5619e-02, -5.2380e-02,  3.3604e-02, -3.2698e-02,  2.5746e-02,\n",
      "         1.8930e-03, -5.2245e-02, -1.1050e-01, -5.2250e-02, -2.1647e-03,\n",
      "        -1.1874e-01,  2.0401e-02,  3.0730e-02, -2.9637e-02, -9.6485e-02,\n",
      "        -4.6408e-03, -3.2503e-02,  4.1557e-03, -1.1865e-02, -6.1439e-03,\n",
      "         4.8034e-02,  2.7082e-03, -9.4860e-02, -2.8149e-02,  3.0365e-03,\n",
      "        -3.7579e-02,  9.8754e-03, -8.9437e-02,  2.4563e-02])), ('bert.encoder.layer.1.output.LayerNorm.weight', tensor([0.7604, 0.7950, 0.7602, 0.8960, 0.6858, 0.6963, 0.7429, 0.7877, 0.6839,\n",
      "        0.7791, 0.7194, 0.9070, 0.8480, 0.7287, 0.7365, 0.7091, 0.6742, 0.7637,\n",
      "        0.7871, 0.7113, 0.7872, 0.7402, 0.7902, 0.7886, 0.8619, 0.8186, 0.8148,\n",
      "        0.8082, 0.8292, 0.7503, 0.8249, 0.7805, 0.7324, 0.7165, 0.7737, 0.7476,\n",
      "        0.7746, 0.7178, 0.7787, 0.8196, 0.7571, 0.6837, 0.7589, 0.7757, 0.8036,\n",
      "        0.7433, 0.7992, 0.7941, 0.6876, 0.8608, 0.8491, 0.8264, 0.8617, 0.8236,\n",
      "        0.8232, 0.8513, 0.7872, 0.7818, 0.8385, 0.8151, 0.8550, 0.8772, 0.8662,\n",
      "        0.8475, 0.8557, 0.8982, 0.9117, 0.9213, 0.9078, 0.8876, 0.9426, 0.9103,\n",
      "        0.9271, 0.9195, 0.7249, 0.9570, 0.9204, 0.9865, 0.9414, 0.9653, 0.7688,\n",
      "        0.5025, 0.9498, 0.9322, 1.0423, 0.8917, 0.9366, 0.9577, 0.8743, 0.8400,\n",
      "        1.0075, 0.9510, 0.9821, 0.8546, 0.9464, 0.9018, 0.9854, 0.9599, 0.9581,\n",
      "        1.0097, 0.9942, 0.8027, 0.9815, 0.9750, 0.9711, 0.7534, 1.0085, 0.9084,\n",
      "        1.0466, 0.9581, 0.9425, 0.9911, 0.9846, 0.9528, 1.0092, 0.9903, 1.0159,\n",
      "        0.9815, 1.0006, 0.8115, 1.0717, 1.0165, 0.9773, 0.9959, 1.0024, 0.9506,\n",
      "        1.0424, 0.8808, 0.9300, 0.9905, 0.7642, 0.9606, 0.9647, 0.9979, 1.0355,\n",
      "        0.9923, 1.0268, 0.9830, 1.0000, 0.9782, 0.9999, 0.9406, 1.0545, 1.0347,\n",
      "        1.0185, 1.0362, 0.9631, 0.9612, 0.9620, 1.0066, 1.0196, 0.9947, 1.0466,\n",
      "        0.9715, 0.8622, 0.9903, 0.8636, 0.8778, 0.9497, 0.3980, 1.0459, 1.0557,\n",
      "        0.9628, 1.0006, 0.9783, 0.7578, 1.0221, 0.8631, 0.8710, 1.0481, 0.9494,\n",
      "        0.9254, 0.9805, 0.9762, 1.0564, 0.9900, 1.0520, 0.9881, 0.9643, 0.8983,\n",
      "        0.9610, 0.8099, 0.9980, 0.9577, 0.9231, 0.9730, 1.0164, 0.7914, 1.0087,\n",
      "        0.9430, 0.9571, 0.9759, 0.9715, 0.9607, 0.8977, 0.9479, 1.0205, 0.9983,\n",
      "        1.0191, 0.9127, 1.0100, 1.0086, 0.9312, 0.9951, 0.9912, 0.9703, 0.8901,\n",
      "        0.9357, 0.9519, 0.9522, 0.9569, 0.9864, 1.0133, 1.0014, 0.9562, 1.0106,\n",
      "        1.0080, 0.9671, 0.7993, 0.9864, 0.9604, 0.9735, 0.7451, 0.9933, 0.8764,\n",
      "        1.0211, 0.9749, 0.9746, 1.0123, 0.9909, 1.0042, 1.0276, 0.9918, 0.9848,\n",
      "        0.9663, 1.0079, 0.8571, 1.0403, 0.9578, 1.0169, 0.9142, 1.0118, 1.0324,\n",
      "        0.9850, 1.0213, 0.8221, 0.8377, 0.8860, 1.0856, 0.9958, 1.0084, 0.9222,\n",
      "        1.0068, 0.9483, 1.0115, 0.9900, 1.0548, 1.0174, 0.8135, 0.9790, 0.9825,\n",
      "        0.9845, 1.0465, 1.0165, 1.0023, 0.9940, 1.0160, 1.0185, 0.9348, 0.9968,\n",
      "        1.0371, 0.9010, 0.9607, 1.0013, 0.9112, 0.9485, 1.0494, 0.9181, 0.9974,\n",
      "        0.8698, 1.0167, 1.0614, 1.0357, 0.9236, 1.0243, 1.0032, 1.0124, 0.7389,\n",
      "        0.9978, 0.9993, 0.9691, 0.9793, 0.9585, 0.9871, 0.9165, 1.0274, 1.0705,\n",
      "        1.0149, 0.9749, 0.8826, 0.9024, 0.9551, 1.0311, 1.0011, 0.9765, 1.0213,\n",
      "        0.9698, 0.9093, 0.9676, 0.9294, 0.9560, 0.9277, 1.0163, 0.9418, 0.9336,\n",
      "        0.9663, 0.9606, 1.0320, 0.9844, 1.0178, 0.9786, 0.9850, 0.9834, 0.9991,\n",
      "        1.0164, 1.0427, 0.9629, 1.0092, 1.0282, 0.9075, 0.9882, 0.9350, 1.0149,\n",
      "        0.8126, 0.8495, 1.0122, 1.0237, 0.9082, 0.9995, 1.0043, 1.0299, 0.9879,\n",
      "        0.9781, 1.0041, 0.9422, 1.0107, 1.0007, 0.9688, 0.8640, 0.9344, 0.9443,\n",
      "        0.9909, 0.9588, 0.9153, 0.9637, 1.0167, 1.0243, 0.9124, 0.9788, 0.9322,\n",
      "        0.9463, 0.9522, 1.0113, 0.7991, 0.9369, 0.8837, 0.9994, 0.9375, 0.9467,\n",
      "        0.8340, 0.9381, 0.9436, 1.0031, 1.0001, 1.0340, 0.9506, 0.9687, 0.9231,\n",
      "        0.9743, 0.9455, 0.8683, 0.9660, 0.9980, 1.0394])), ('bert.encoder.layer.1.output.LayerNorm.bias', tensor([ 0.0553, -0.0079, -0.0765,  0.0016, -0.0657,  0.0691,  0.0743, -0.0050,\n",
      "        -0.0136,  0.0883,  0.1575,  0.0020, -0.0586,  0.1723,  0.1043,  0.0296,\n",
      "         0.1142, -0.0081, -0.0224, -0.0687,  0.0497,  0.1003,  0.1201, -0.1709,\n",
      "        -0.0509, -0.0459, -0.0010, -0.0676, -0.0937,  0.1419,  0.0400,  0.0201,\n",
      "        -0.0007, -0.0355, -0.0653,  0.0199, -0.0139,  0.1573,  0.0279,  0.0126,\n",
      "         0.0927,  0.0840,  0.0313,  0.0244, -0.1078,  0.1344,  0.0868, -0.0564,\n",
      "         0.1427,  0.0161,  0.0164, -0.0285, -0.0292,  0.0233, -0.0177,  0.0687,\n",
      "         0.0782, -0.0031,  0.0963,  0.0262,  0.0635,  0.0615, -0.1002, -0.0243,\n",
      "         0.1348,  0.0738, -0.0359,  0.0150, -0.0399, -0.0968, -0.0074,  0.0570,\n",
      "         0.0051,  0.0862,  0.2479, -0.0720, -0.0522, -0.0309,  0.0379,  0.0460,\n",
      "        -0.2033, -0.3200, -0.0290, -0.0308,  0.0361, -0.0359,  0.0195, -0.0604,\n",
      "         0.0379, -0.0527,  0.0056,  0.0414,  0.0463, -0.1099, -0.0042, -0.1167,\n",
      "         0.0015, -0.0073,  0.0550, -0.0034,  0.0639, -0.0663,  0.0234,  0.0299,\n",
      "         0.0455, -0.1237, -0.0061, -0.0603,  0.0087,  0.0139, -0.0294, -0.0240,\n",
      "         0.0501, -0.0698, -0.0089,  0.0159,  0.0148,  0.0061,  0.0052,  0.0036,\n",
      "        -0.0735,  0.0318, -0.0418, -0.0748,  0.0061,  0.1165, -0.0005, -0.0608,\n",
      "         0.0596, -0.0435, -0.1500,  0.0545,  0.0801,  0.0531,  0.0525,  0.0065,\n",
      "         0.0152,  0.0118,  0.0377, -0.0224,  0.0136, -0.0186, -0.0071, -0.0017,\n",
      "         0.0367,  0.0093,  0.0809,  0.0123, -0.0050, -0.0163, -0.0098,  0.0300,\n",
      "         0.0244,  0.0276, -0.0573, -0.0634,  0.0856, -0.0629,  0.0335, -0.2795,\n",
      "         0.0343, -0.0262,  0.0593, -0.0354,  0.0536, -0.1531, -0.0256, -0.1350,\n",
      "         0.0313,  0.0016, -0.0232,  0.0454,  0.0013, -0.0407,  0.0553, -0.0031,\n",
      "         0.0079, -0.0033,  0.0110, -0.0374,  0.0386, -0.0917, -0.0324,  0.0267,\n",
      "         0.0356,  0.1140,  0.0520, -0.0751,  0.0487, -0.1308,  0.0087,  0.0314,\n",
      "         0.0540, -0.0412,  0.0945, -0.0189,  0.0041, -0.0257,  0.0050, -0.0651,\n",
      "         0.0817,  0.0061,  0.0836,  0.0318,  0.0623,  0.0197, -0.0028, -0.0540,\n",
      "         0.0867, -0.0183,  0.0017,  0.0605,  0.0466, -0.0154, -0.0061, -0.0065,\n",
      "         0.0307,  0.0535,  0.0761,  0.0014,  0.0893,  0.0266,  0.0990,  0.0504,\n",
      "         0.0513, -0.0552,  0.0207, -0.0538,  0.0171,  0.0320,  0.0713, -0.0122,\n",
      "         0.0567, -0.0314,  0.0052, -0.0482,  0.0335, -0.0202, -0.0088,  0.0271,\n",
      "         0.0805, -0.0189,  0.0132, -0.0068, -0.0070, -0.0844,  0.1281, -0.0399,\n",
      "        -0.0287,  0.0274,  0.0195, -0.0244,  0.0388,  0.0454,  0.0224, -0.0275,\n",
      "         0.0031, -0.0440,  0.1474,  0.0208, -0.0076,  0.0729,  0.0587,  0.0353,\n",
      "         0.0136, -0.0186, -0.0074, -0.0105,  0.1178, -0.0281, -0.0051, -0.0304,\n",
      "         0.0343, -0.0075,  0.0639, -0.0090,  0.0087, -0.0652, -0.0148, -0.0301,\n",
      "        -0.0224,  0.0243,  0.0496,  0.0239,  0.0543,  0.0115,  0.0583, -0.0743,\n",
      "         0.0247, -0.0217,  0.0512, -0.0552,  0.0440, -0.0570,  0.1018,  0.0245,\n",
      "        -0.0195,  0.0369,  0.0424, -0.0953,  0.0964,  0.0219,  0.0119, -0.0174,\n",
      "         0.0666,  0.0714,  0.0637, -0.0559,  0.0680, -0.0799,  0.0564,  0.0717,\n",
      "         0.0225,  0.0140,  0.0315,  0.0296,  0.0243,  0.0216, -0.0213, -0.0356,\n",
      "         0.0134,  0.0449,  0.1233, -0.0527,  0.0296, -0.0288,  0.0678, -0.0151,\n",
      "         0.0157,  0.0018,  0.0587, -0.0717,  0.0402, -0.1355,  0.0230, -0.0011,\n",
      "        -0.0103, -0.0484, -0.0018, -0.0100, -0.0244, -0.0641,  0.0521, -0.0370,\n",
      "         0.0537, -0.0145,  0.0043, -0.0507,  0.0928, -0.0124,  0.0632,  0.0842,\n",
      "         0.0459, -0.0378,  0.0182, -0.0775,  0.0632, -0.0974,  0.0369, -0.0199,\n",
      "        -0.0235,  0.0146,  0.0299, -0.0077, -0.0851,  0.0372,  0.0172, -0.0843,\n",
      "        -0.0165, -0.1561,  0.0571,  0.0056,  0.0432,  0.0478,  0.0490,  0.0279,\n",
      "         0.0207, -0.1104,  0.0290, -0.0664, -0.0486,  0.0384,  0.0255,  0.0384])), ('bert.encoder.layer.2.attention.self.query.weight', tensor([[ 0.0240,  0.2321, -0.1960,  ...,  0.0522, -0.0810, -0.0260],\n",
      "        [-0.3311, -0.1914,  0.1465,  ...,  0.0490, -0.0273, -0.0789],\n",
      "        [ 0.1309,  0.3873, -0.5473,  ..., -0.1215,  0.0972, -0.0465],\n",
      "        ...,\n",
      "        [-0.0256, -0.1399,  0.0156,  ...,  0.0257,  0.0033,  0.0511],\n",
      "        [ 0.0554,  0.0858, -0.0723,  ..., -0.0349,  0.0264, -0.0060],\n",
      "        [ 0.0210,  0.0654,  0.1206,  ...,  0.0030,  0.0915, -0.0060]])), ('bert.encoder.layer.2.attention.self.query.bias', tensor([-3.2088e-02,  1.3578e-01, -4.6778e-02, -7.1279e-03,  1.3480e-01,\n",
      "        -1.0574e-01,  3.0677e-02,  1.4060e-01, -5.3156e-03, -3.8388e-02,\n",
      "        -4.6412e-02,  1.3777e-01, -2.8219e-02,  1.0985e-02, -9.9537e-02,\n",
      "        -2.2366e-02,  3.7929e-02,  7.6063e-02,  1.0510e-01,  6.8123e-02,\n",
      "        -3.6487e-03,  4.5163e-02, -7.5858e-02,  9.1975e-02,  1.2115e-02,\n",
      "        -1.2543e-02,  8.2427e-02,  1.2732e-02,  7.0588e-02,  3.9911e-02,\n",
      "         4.2453e-02, -5.4279e-02,  3.0905e-02,  6.6514e-02, -6.3353e-02,\n",
      "         7.8095e-02,  7.8636e-02, -6.7280e-02,  6.2291e-02, -9.9903e-02,\n",
      "         5.6469e-02, -1.5274e-02,  5.8351e-02,  5.1076e-02,  1.0731e-01,\n",
      "        -9.4277e-03, -1.6488e-02, -6.8587e-02, -2.1305e-02, -2.5372e-03,\n",
      "        -3.4659e-02,  7.2761e-02, -4.6547e-03, -8.8853e-02, -1.2805e-01,\n",
      "        -2.7343e-03,  3.5220e-02, -8.2383e-02, -1.7228e-02, -1.1065e-01,\n",
      "        -1.9581e-02, -3.9254e-02, -1.0097e-01, -2.0689e-02,  1.6642e-01,\n",
      "         1.9346e-01,  1.1735e-01, -1.5490e-01, -4.7810e-02, -3.2777e-02,\n",
      "         4.1058e-02,  7.9355e-02,  8.8950e-02,  3.6151e-02,  9.0690e-02,\n",
      "        -1.1939e-01, -4.6487e-02, -9.2831e-03, -1.4223e-02, -9.2698e-02,\n",
      "        -1.2251e-02,  2.4043e-02, -1.2579e-02, -3.5777e-02,  6.9738e-02,\n",
      "        -6.6056e-02, -2.1670e-01,  4.4197e-02,  1.0419e-02, -7.9363e-03,\n",
      "         3.6980e-02,  2.2592e-02,  1.4556e-01,  6.5973e-02,  1.3320e-02,\n",
      "        -1.1986e-01,  2.1213e-02, -9.9802e-03,  1.0971e-01, -8.0290e-03,\n",
      "         1.4980e-02,  7.7607e-03, -6.1506e-02, -6.4788e-02,  4.6181e-03,\n",
      "         3.2243e-02,  3.7262e-02, -2.3727e-03, -1.1203e-01, -4.4743e-02,\n",
      "        -1.8017e-02, -3.5137e-02,  6.6291e-03,  9.2999e-02, -3.0284e-02,\n",
      "         2.2465e-02, -1.2753e-02,  3.7132e-03, -1.5156e-02,  9.7414e-02,\n",
      "         4.6030e-02,  5.9347e-02,  6.8648e-02, -9.1975e-02,  2.6284e-02,\n",
      "        -2.2776e-02, -3.3756e-02,  2.4184e-03,  9.7459e-02, -9.1336e-02,\n",
      "        -2.8161e-02, -5.0011e-02,  9.5933e-02,  7.3207e-02,  7.4726e-02,\n",
      "         5.6011e-02,  4.7007e-02,  2.7535e-02, -1.7231e-02,  6.8397e-02,\n",
      "        -9.7977e-02, -5.0677e-02, -5.3781e-02, -8.7942e-03, -4.1273e-03,\n",
      "         3.5763e-02, -1.0978e-03,  6.6945e-02, -6.4643e-02,  6.0381e-02,\n",
      "        -6.5707e-02,  7.0059e-02, -4.3521e-02,  5.1619e-03,  6.5728e-02,\n",
      "         1.0706e-01, -5.8458e-02, -4.9527e-02, -4.8259e-02, -5.1339e-02,\n",
      "        -4.0768e-03,  8.6331e-03,  2.3158e-02,  3.9990e-03,  7.2179e-02,\n",
      "         1.7666e-02,  2.0961e-02,  1.2423e-02,  4.0752e-03, -1.9736e-03,\n",
      "        -2.5776e-02,  5.8684e-03, -2.9965e-02,  2.4870e-03, -2.9165e-02,\n",
      "         6.6085e-03, -9.4293e-03, -4.4806e-03,  7.0743e-03, -2.0855e-02,\n",
      "        -2.8606e-02,  7.7836e-03,  1.1144e-02,  4.5203e-02,  4.1180e-03,\n",
      "        -2.8341e-02, -8.7642e-03,  1.2826e-03,  3.4892e-02,  1.1619e-02,\n",
      "        -1.9055e-02,  9.4837e-03,  3.9432e-02, -3.2632e-03, -3.1158e-04,\n",
      "        -6.0332e-02, -5.4158e-02,  6.2880e-03,  8.4537e-03, -3.2110e-02,\n",
      "         7.0724e-02, -1.9067e-03, -1.3576e-01, -7.6722e-02,  6.0154e-02,\n",
      "        -1.9181e-02, -7.5361e-02,  1.1904e-01,  1.9822e-01, -8.0734e-02,\n",
      "         7.1314e-03, -2.1756e-04, -1.0253e-03, -1.4406e-01, -7.3064e-02,\n",
      "         6.0438e-02,  4.9313e-02,  5.4104e-02, -4.8444e-02, -1.5809e-01,\n",
      "         1.1988e-01, -3.3090e-02,  1.2998e-01, -5.9694e-02, -6.5663e-02,\n",
      "        -9.5200e-03, -5.9729e-02,  5.2042e-02,  2.5490e-03,  1.2470e-01,\n",
      "        -2.8839e-02, -4.8861e-02,  2.4824e-02,  3.7639e-02, -1.5644e-04,\n",
      "         1.5388e-02, -8.3033e-03,  1.8313e-02, -6.8657e-02, -2.0045e-02,\n",
      "        -2.0768e-02,  2.0975e-02,  2.5606e-02,  8.5121e-02,  4.6728e-03,\n",
      "         5.5760e-03,  5.1648e-02, -3.3141e-02, -1.8837e-02,  3.7805e-02,\n",
      "        -3.5691e-02,  3.7111e-02, -6.7048e-03,  7.5602e-02,  2.1126e-02,\n",
      "         2.7608e-02, -3.6263e-02,  1.2261e-01, -6.2887e-02,  1.5599e-02,\n",
      "         1.1966e-01, -2.2329e-02, -1.1477e-01, -4.5358e-02, -2.0557e-02,\n",
      "         2.9564e-02,  2.0852e-02, -3.9243e-02,  5.2636e-02,  8.9222e-03,\n",
      "        -2.5069e-02, -6.7059e-02, -1.4865e-02,  5.9207e-03,  8.7126e-03,\n",
      "        -1.8128e-03, -1.1716e-01, -6.1581e-03,  5.5422e-02,  1.2901e-01,\n",
      "        -4.3517e-02, -1.0275e-01, -4.1499e-02,  3.8353e-03, -4.0321e-02,\n",
      "         4.7869e-02,  3.5432e-02,  6.4246e-02, -5.4882e-02, -1.3354e-01,\n",
      "         7.8391e-02, -3.0888e-02, -2.3309e-03,  6.7544e-02,  7.2568e-02,\n",
      "        -4.0328e-03, -1.2233e-01, -2.9341e-02,  2.0457e-02,  1.3271e-02,\n",
      "        -8.1922e-02, -2.7779e-02, -7.6819e-04,  6.9440e-02,  5.1901e-02,\n",
      "        -9.6782e-02, -1.2993e-01,  4.2391e-03,  4.9886e-02,  1.0904e-02,\n",
      "        -2.5228e-02,  6.4858e-02,  6.2521e-02, -1.1933e-01,  7.2653e-02,\n",
      "        -6.2013e-02, -4.6464e-02,  6.6923e-02,  1.2700e-01,  1.2978e-01,\n",
      "        -4.1201e-02,  1.0227e-01, -1.0606e-01,  1.2031e-01, -1.2358e-01,\n",
      "         1.2553e-01, -5.1218e-02, -4.4611e-02,  9.5337e-02,  3.0741e-02,\n",
      "        -9.3411e-02,  1.6441e-01, -3.6436e-02,  2.3613e-02, -9.3764e-02,\n",
      "        -2.8258e-02, -3.7018e-02, -1.2480e-01,  1.2396e-02,  5.7808e-02,\n",
      "        -2.7422e-02,  3.0368e-02, -7.3952e-02,  3.6440e-02, -3.1195e-02,\n",
      "        -1.2470e-01,  5.1322e-02, -8.7648e-03, -7.4268e-03,  6.2376e-02,\n",
      "         1.5852e-01,  1.4269e-01,  8.4698e-02,  2.3199e-02,  1.5090e-01,\n",
      "         3.7822e-02,  1.0119e-01,  1.4988e-01, -3.3423e-02, -8.5891e-02,\n",
      "        -1.8176e-02,  7.6101e-02,  2.0445e-02, -7.2836e-02,  9.5017e-02,\n",
      "         7.9988e-02,  4.3256e-02,  8.1557e-05, -4.0714e-02,  1.2540e-02,\n",
      "        -6.7154e-02,  9.4938e-03,  6.4573e-02, -8.6986e-02,  3.0886e-02,\n",
      "        -1.5968e-01,  1.2829e-02,  2.0687e-02, -2.4253e-02, -1.4199e-01,\n",
      "        -1.5170e-01, -6.2758e-02,  6.5640e-02, -1.4481e-01])), ('bert.encoder.layer.2.attention.self.key.weight', tensor([[ 0.2322,  0.0435,  0.1028,  ..., -0.0807, -0.0262, -0.0779],\n",
      "        [ 0.0923, -0.0228, -0.2185,  ...,  0.0783, -0.0273,  0.0669],\n",
      "        [ 0.2329,  0.3699, -0.1487,  ...,  0.0206, -0.0906, -0.0597],\n",
      "        ...,\n",
      "        [ 0.0075, -0.0140,  0.0977,  ..., -0.0774,  0.0171,  0.0075],\n",
      "        [-0.1729, -0.0343,  0.0311,  ..., -0.0252,  0.0507, -0.0527],\n",
      "        [ 0.0335,  0.0434,  0.1049,  ..., -0.0364,  0.0433,  0.0157]])), ('bert.encoder.layer.2.attention.self.key.bias', tensor([ 6.1393e-04,  1.0120e-02, -1.0001e-02, -2.0796e-02,  1.7151e-02,\n",
      "        -2.4013e-02,  1.6031e-03,  3.7697e-02,  2.4811e-03, -1.3199e-04,\n",
      "        -2.5262e-03,  2.7421e-03, -3.4250e-03,  1.1366e-02, -3.3368e-02,\n",
      "         2.3042e-02,  3.1984e-02, -4.0899e-03, -2.5869e-02, -7.3976e-04,\n",
      "        -1.8273e-02,  4.1201e-03, -1.8297e-03,  1.0742e-02, -1.9009e-03,\n",
      "         3.0334e-02,  1.8623e-02,  1.3386e-02,  2.4637e-03,  4.8776e-03,\n",
      "         2.4999e-02,  7.3454e-03,  2.6976e-03,  3.4401e-04, -2.1274e-04,\n",
      "        -5.9061e-03, -1.5955e-03,  9.7844e-04, -1.6030e-05, -1.1196e-04,\n",
      "         2.1389e-04,  1.1913e-03, -4.4133e-04, -6.3169e-03, -2.9078e-03,\n",
      "        -1.5839e-03, -1.0343e-03, -5.2027e-04, -2.4528e-04, -5.6319e-03,\n",
      "        -5.9598e-04, -2.4751e-03,  2.2399e-04,  3.3619e-03, -2.9357e-03,\n",
      "        -2.2446e-03,  1.1969e-03, -8.3186e-04, -1.9927e-03,  6.8376e-04,\n",
      "         2.4558e-03,  9.2203e-03, -2.4238e-04,  1.4787e-03,  2.5183e-03,\n",
      "        -2.4717e-03, -4.3362e-03, -4.7321e-03,  3.6035e-03, -1.3726e-03,\n",
      "        -4.6081e-03,  8.3612e-04, -5.0918e-03, -4.7327e-04, -2.6305e-03,\n",
      "        -3.9383e-04, -1.4032e-03,  2.2766e-03, -1.3840e-03,  3.4571e-03,\n",
      "         1.8402e-03,  8.1190e-04,  7.3804e-04, -5.0188e-03, -8.4522e-04,\n",
      "         1.4332e-03,  7.4338e-04,  6.0082e-03,  1.4396e-03, -4.2242e-03,\n",
      "         1.2893e-03, -9.2124e-04,  3.9722e-04, -2.4699e-03, -1.8623e-03,\n",
      "         1.4202e-04,  1.5118e-03,  6.2159e-04, -3.3455e-04, -5.4132e-04,\n",
      "        -1.3966e-04, -1.2467e-03, -1.4216e-04,  6.1950e-05, -7.2392e-04,\n",
      "         9.1543e-04,  5.5788e-04,  8.7500e-04,  3.5581e-03,  1.8159e-03,\n",
      "        -1.9975e-03,  2.9855e-03,  3.0435e-05, -2.2146e-03,  2.9884e-03,\n",
      "        -8.6315e-04,  8.5614e-05, -3.4318e-04, -5.3710e-04,  1.6387e-03,\n",
      "        -1.0536e-03, -1.5550e-04, -1.1259e-03,  3.5665e-03, -4.3447e-04,\n",
      "        -2.7385e-03,  3.9666e-04, -5.2865e-04,  2.8812e-03, -4.1795e-04,\n",
      "        -1.7825e-03, -2.4014e-05, -1.6335e-03,  1.0021e-03,  1.2382e-03,\n",
      "         1.9935e-03,  2.2152e-03, -1.0674e-03,  7.6577e-04,  7.2558e-04,\n",
      "        -4.1654e-03,  5.5636e-04, -7.1125e-04, -1.0573e-03, -4.0462e-03,\n",
      "         3.2768e-03,  2.5236e-05,  1.2668e-03, -1.0895e-03,  2.2632e-03,\n",
      "         4.6444e-04, -7.2887e-04, -3.0736e-05,  4.0384e-03, -3.3664e-04,\n",
      "         3.6924e-03, -1.4826e-03, -1.7762e-03, -1.2602e-03, -2.1623e-03,\n",
      "         1.7695e-03,  1.1102e-04, -1.6091e-03, -8.8854e-05, -4.6193e-03,\n",
      "         2.3930e-03,  1.6881e-03, -5.4712e-03, -1.6490e-03, -2.3139e-03,\n",
      "         6.3155e-04, -2.1823e-04, -1.2762e-03, -1.0520e-03, -1.5907e-04,\n",
      "         4.2423e-04, -1.4654e-03,  1.9379e-03, -1.3080e-03,  2.3322e-03,\n",
      "        -1.1515e-04, -9.3248e-05, -1.6459e-03,  4.2310e-04,  2.2366e-03,\n",
      "         1.9393e-03, -5.1604e-04, -2.0834e-03,  2.5450e-03, -1.0497e-03,\n",
      "        -1.2585e-03, -2.3418e-03, -9.8510e-04,  3.5040e-03, -4.6843e-03,\n",
      "        -1.8246e-03,  2.3944e-03, -3.6535e-03, -7.7093e-04, -4.6628e-04,\n",
      "        -1.6583e-03, -1.9238e-03,  9.8591e-04, -6.3287e-04,  3.4953e-03,\n",
      "         2.6740e-03,  3.3010e-03, -6.3213e-04,  1.4883e-03,  5.5251e-03,\n",
      "         1.3716e-03, -2.8074e-03,  6.4120e-04, -5.1493e-03,  4.8607e-03,\n",
      "        -2.6835e-03, -3.5698e-03,  1.9049e-03, -2.7856e-03, -1.6919e-03,\n",
      "        -1.4636e-03,  2.5626e-03, -4.0507e-04,  3.4082e-03, -1.1755e-03,\n",
      "        -1.7166e-03,  4.4904e-05,  2.7718e-03, -6.5635e-04,  2.2007e-04,\n",
      "         5.3984e-04, -1.9612e-03, -1.0450e-03, -4.6993e-04, -1.5149e-03,\n",
      "        -1.1491e-03,  1.3122e-03, -6.6961e-04,  2.6504e-04, -2.6004e-04,\n",
      "         2.6220e-04,  8.7455e-04, -4.6908e-06,  7.8797e-04,  9.0563e-04,\n",
      "         1.0289e-03, -1.4890e-03, -3.9074e-04,  1.1336e-03, -3.8177e-04,\n",
      "        -5.1756e-04,  1.4010e-04,  1.6820e-05,  1.9840e-03, -4.0053e-04,\n",
      "         8.1847e-04, -6.5361e-04,  2.7555e-03, -1.0413e-03,  2.4151e-04,\n",
      "         1.3624e-03, -7.8560e-04,  8.1503e-04, -8.1299e-04, -7.8473e-04,\n",
      "        -6.1467e-04, -7.5276e-04, -4.7775e-05,  8.3616e-04, -1.0125e-04,\n",
      "         4.4837e-04,  5.5501e-04,  1.3426e-03,  1.8964e-03,  1.3999e-04,\n",
      "        -1.5497e-03, -1.7275e-03,  1.5846e-03,  2.4043e-04, -3.4280e-04,\n",
      "        -5.9943e-05,  7.6550e-04,  1.0009e-04,  7.1086e-04,  9.2497e-04,\n",
      "         2.2583e-05,  5.8227e-05,  2.0367e-04, -3.6948e-03,  9.9588e-04,\n",
      "         6.0111e-05, -2.0214e-03,  2.2698e-03, -1.1456e-04,  3.3100e-03,\n",
      "        -2.1037e-03, -2.7378e-03, -1.2759e-03,  1.7980e-03, -3.4163e-03,\n",
      "        -6.6153e-04, -2.1021e-03, -7.0540e-03, -4.2491e-03,  7.0992e-04,\n",
      "        -2.6459e-03,  2.1323e-03,  1.7081e-03, -7.2435e-04, -4.4794e-03,\n",
      "         3.2389e-03, -5.2923e-04, -1.6892e-03,  3.0924e-03, -1.4339e-03,\n",
      "         4.5460e-03,  4.9819e-03,  3.1157e-04, -3.5448e-03,  6.2280e-04,\n",
      "        -2.0217e-02,  1.6736e-03, -3.5228e-02,  3.7743e-02, -5.1600e-02,\n",
      "         6.6514e-03, -5.1760e-02,  1.4460e-02, -4.8038e-03, -1.9416e-02,\n",
      "        -4.6211e-04,  2.6810e-02, -2.3906e-02, -3.5438e-03, -6.2471e-03,\n",
      "         6.3736e-02, -1.0164e-02,  9.7960e-03, -4.0480e-03,  1.8505e-02,\n",
      "        -7.3808e-03, -4.1854e-02,  1.8311e-03,  2.8329e-02, -6.3972e-02,\n",
      "        -1.2418e-02, -2.6440e-02,  5.8507e-03,  1.5181e-02,  1.3443e-02,\n",
      "         3.0861e-02,  1.2993e-02,  2.3667e-03, -1.0693e-03,  2.7201e-03,\n",
      "        -3.2979e-03, -3.9096e-03, -2.6287e-03,  1.6545e-03, -1.8874e-03,\n",
      "         3.4811e-05,  2.8471e-03, -1.5790e-03, -3.1395e-03, -6.3548e-03,\n",
      "         5.8768e-03, -1.3365e-03, -3.7755e-03, -4.7046e-04, -7.0970e-04,\n",
      "         4.5832e-04, -5.2907e-04, -3.2227e-03, -3.4981e-03, -6.7856e-04,\n",
      "         6.4568e-04,  6.6393e-03, -3.6743e-03, -1.9868e-03, -8.3500e-04,\n",
      "        -4.0356e-03, -4.3416e-04, -1.1792e-03, -8.0378e-05])), ('bert.encoder.layer.2.attention.self.value.weight', tensor([[ 0.0120,  0.0334, -0.0019,  ..., -0.0137, -0.0273, -0.0580],\n",
      "        [-0.0029,  0.0072,  0.0216,  ...,  0.0190,  0.0243,  0.0150],\n",
      "        [-0.0865,  0.0495, -0.0259,  ...,  0.0626, -0.0874, -0.1042],\n",
      "        ...,\n",
      "        [-0.0176,  0.0751, -0.0182,  ...,  0.0178, -0.0250, -0.0134],\n",
      "        [-0.0411,  0.0519,  0.0229,  ...,  0.0182,  0.0665,  0.0488],\n",
      "        [-0.0184,  0.0406, -0.0462,  ...,  0.0265, -0.0432, -0.0094]])), ('bert.encoder.layer.2.attention.self.value.bias', tensor([ 1.5020e-02,  1.0290e-02, -3.8505e-02,  2.7848e-02,  8.4457e-02,\n",
      "        -4.3249e-02,  3.5238e-02, -3.6327e-02, -6.2050e-02, -5.4399e-02,\n",
      "         1.8114e-02,  3.5149e-04, -2.3939e-02, -2.3505e-02, -2.2628e-02,\n",
      "         5.3178e-02,  2.8540e-02,  8.8681e-02,  1.3661e-02,  5.2673e-02,\n",
      "        -1.4726e-02, -7.0978e-02, -3.1702e-02,  1.6592e-01,  1.3622e-01,\n",
      "         1.2948e-01, -2.1574e-02,  6.1469e-02,  6.2070e-02, -1.1608e-01,\n",
      "         1.2374e-02, -4.1438e-02, -4.2275e-02,  6.0572e-02, -3.3835e-03,\n",
      "         7.8473e-03, -1.2792e-03, -4.2917e-02, -1.9913e-04,  6.8898e-03,\n",
      "         2.6541e-03, -6.8924e-02, -3.4635e-02,  6.1047e-02, -3.2276e-02,\n",
      "        -3.1147e-02, -6.9129e-02, -6.4127e-02,  1.1099e-02, -4.4648e-02,\n",
      "         1.0887e-02,  2.9142e-02, -1.2313e-02,  1.0087e-02,  5.8475e-02,\n",
      "         2.1694e-02,  4.0970e-02, -2.7227e-02,  9.3473e-02,  4.1793e-02,\n",
      "         7.0116e-02,  2.8273e-02,  6.2445e-03,  3.5719e-02,  1.0478e-02,\n",
      "        -2.3075e-02,  7.6113e-02,  1.4658e-02,  3.9952e-02,  1.0897e-02,\n",
      "        -2.6392e-03,  2.1795e-03, -1.5279e-02, -1.8162e-02,  9.2324e-03,\n",
      "         9.9919e-02, -1.9198e-02,  6.4616e-03, -2.7225e-02, -2.9406e-02,\n",
      "         3.2636e-02,  7.1137e-02,  3.4403e-02, -1.0266e-01, -3.9005e-02,\n",
      "         7.9683e-02, -6.5391e-02, -1.9951e-02,  1.6791e-02,  3.6896e-02,\n",
      "         5.4427e-02, -6.2535e-03,  7.2067e-02,  3.5800e-02,  1.0062e-01,\n",
      "         1.8909e-02,  1.4588e-02,  1.2531e-02, -2.5356e-03, -1.5145e-02,\n",
      "        -3.2772e-03, -6.1351e-03,  6.0037e-03, -1.0231e-02,  1.8867e-02,\n",
      "         3.3090e-02,  1.2015e-02,  4.9625e-05,  3.4055e-02, -4.4992e-03,\n",
      "        -1.2595e-02,  6.0769e-03,  2.0916e-02,  1.9545e-02, -9.2965e-03,\n",
      "         6.6438e-03,  1.4874e-02, -9.8816e-04,  1.0749e-02, -5.4526e-03,\n",
      "         3.4487e-05, -3.4052e-04, -2.3678e-03, -1.9349e-02,  3.0133e-02,\n",
      "        -2.3112e-03,  8.7738e-03, -7.8903e-03,  6.2053e-03, -2.4076e-02,\n",
      "         1.9059e-02,  1.3467e-02, -3.0412e-02,  2.4161e-02, -6.1448e-03,\n",
      "        -3.7842e-02, -4.3712e-02,  1.8415e-02, -1.2572e-02,  1.0511e-02,\n",
      "        -1.0769e-02, -7.7984e-03, -9.0669e-03, -1.3486e-02, -1.0360e-02,\n",
      "         1.0159e-02,  1.4995e-02,  5.7953e-04, -7.5073e-03, -7.4375e-04,\n",
      "         1.9148e-02,  1.4921e-02,  2.0225e-02, -4.1437e-02,  4.3623e-03,\n",
      "        -2.7638e-03, -3.4640e-02,  5.4006e-03,  2.9038e-02,  3.8835e-02,\n",
      "         9.1531e-03, -3.8366e-04,  2.7602e-02, -9.4981e-03, -7.3857e-03,\n",
      "         3.3278e-03,  7.3553e-03,  1.8442e-02,  1.5216e-02, -1.3296e-02,\n",
      "         5.4117e-04, -6.5093e-03,  5.3234e-03, -8.3522e-03,  9.4196e-03,\n",
      "         6.4844e-04,  5.9159e-03, -1.4918e-02, -6.9671e-03,  7.8061e-03,\n",
      "        -2.0340e-02, -7.5198e-03, -2.9491e-03,  1.4073e-02, -2.7759e-02,\n",
      "        -8.5059e-04,  1.2430e-02,  1.5778e-02, -1.4851e-02, -9.6558e-03,\n",
      "        -9.7398e-03,  1.4299e-03, -5.1665e-04, -2.3791e-02,  4.4110e-03,\n",
      "        -2.1019e-03, -5.2015e-02,  1.0427e-02, -1.9570e-02,  2.5998e-02,\n",
      "         2.6469e-02, -3.4515e-02, -3.0833e-03, -2.6950e-03, -3.0170e-02,\n",
      "         3.0794e-02,  3.8004e-03, -3.0807e-02,  2.5438e-04, -2.9514e-02,\n",
      "        -8.4736e-03, -1.3294e-02,  3.7973e-02,  4.1248e-02,  1.3731e-02,\n",
      "         2.5549e-02, -1.6069e-02, -2.0161e-02,  3.4095e-02,  1.1912e-02,\n",
      "        -2.3876e-02,  1.8895e-02, -5.5857e-02,  6.6216e-04, -6.7319e-03,\n",
      "        -2.0669e-03,  2.6573e-02,  1.1374e-02, -2.4073e-03, -1.1666e-02,\n",
      "        -5.6802e-04,  7.8734e-03,  3.2679e-02,  2.0961e-03,  1.2602e-02,\n",
      "        -5.7988e-03,  1.3462e-02,  1.6218e-02, -7.4508e-03, -1.7888e-02,\n",
      "         3.1691e-02, -7.7160e-03,  2.8029e-02,  4.0559e-03,  1.5518e-02,\n",
      "        -2.2593e-03, -1.1780e-02, -2.3125e-02, -5.4783e-02, -1.0139e-02,\n",
      "         1.3488e-02, -9.1043e-03, -9.0742e-03,  2.3956e-02, -1.7516e-02,\n",
      "         3.4595e-02, -4.1909e-03,  1.8008e-02,  1.4703e-02, -1.5364e-02,\n",
      "        -1.2480e-02,  1.2604e-03, -2.4166e-02,  4.6624e-03,  8.7751e-03,\n",
      "         2.4500e-02,  7.3259e-03, -7.3837e-03,  4.5903e-03,  3.1198e-03,\n",
      "        -7.0717e-03,  2.8712e-02, -5.1579e-03,  6.7398e-04,  1.4454e-02,\n",
      "         2.3337e-03, -1.5811e-03, -1.9528e-02, -5.4953e-03, -4.1935e-02,\n",
      "         2.4367e-02,  8.1188e-03, -1.6357e-02,  2.1505e-03, -4.5020e-03,\n",
      "         1.9815e-02,  4.8817e-03, -1.4894e-02,  4.3642e-03, -2.3721e-03,\n",
      "         1.0296e-02,  4.9245e-03,  2.2189e-02, -5.1312e-03,  7.1785e-04,\n",
      "         2.0004e-02,  5.1127e-03,  2.7455e-02,  4.3684e-03, -2.5392e-02,\n",
      "        -9.5807e-04, -8.4115e-03,  1.3179e-02, -2.6051e-04, -1.2870e-02,\n",
      "         8.1318e-03,  2.5204e-04, -1.1981e-02,  1.5279e-02, -1.2576e-02,\n",
      "         8.0496e-03, -3.1322e-02, -7.0195e-03,  8.8060e-03, -4.4107e-03,\n",
      "        -1.5144e-02,  1.6598e-02, -1.6558e-02, -1.6002e-02, -1.2677e-03,\n",
      "         2.7290e-02,  5.0918e-02,  1.7654e-02, -2.8109e-02, -3.3768e-02,\n",
      "        -6.6489e-02,  2.2652e-02, -1.7423e-02, -1.2903e-02,  6.6791e-02,\n",
      "        -2.8022e-02,  7.1456e-02,  3.8937e-02, -3.8808e-02, -1.2152e-01,\n",
      "        -5.6108e-02,  8.4863e-02,  1.6201e-02,  8.6461e-03,  9.8536e-02,\n",
      "        -6.9669e-02, -4.0653e-02,  4.7595e-02, -8.0618e-02,  3.4261e-02,\n",
      "        -3.7872e-02, -1.5663e-02, -2.4941e-02, -3.4655e-02,  2.0189e-01,\n",
      "         6.8660e-02, -2.3408e-02, -1.0412e-02,  3.6095e-02,  2.5012e-03,\n",
      "         7.5470e-03,  6.3649e-03, -2.4554e-02, -2.2800e-02,  9.5187e-03,\n",
      "        -4.6707e-03,  2.0953e-02,  1.8321e-02, -1.2701e-03,  2.2467e-02,\n",
      "         2.5837e-02,  1.3938e-02, -4.8883e-02,  4.6904e-02, -4.3598e-02,\n",
      "        -2.3203e-02,  2.7033e-02,  5.0039e-03, -1.6407e-02,  2.4997e-02,\n",
      "         1.7664e-02, -2.1451e-03, -3.9586e-02, -5.0474e-03,  3.7122e-02,\n",
      "         1.9992e-02, -1.5236e-02,  1.1377e-02,  1.0543e-03])), ('bert.encoder.layer.2.attention.output.dense.weight', tensor([[ 0.0180, -0.0158, -0.0483,  ..., -0.1394,  0.0702,  0.0048],\n",
      "        [ 0.1334,  0.0587,  0.0533,  ...,  0.1549,  0.0236,  0.1198],\n",
      "        [ 0.0065, -0.0159,  0.0357,  ..., -0.1144, -0.0465, -0.1702],\n",
      "        ...,\n",
      "        [-0.0111,  0.0748,  0.0040,  ..., -0.0029,  0.0372, -0.0832],\n",
      "        [-0.0046, -0.0150, -0.0420,  ..., -0.0005,  0.0541,  0.0376],\n",
      "        [-0.0022, -0.0067,  0.0115,  ..., -0.0195, -0.0224, -0.0084]])), ('bert.encoder.layer.2.attention.output.dense.bias', tensor([-6.1439e-02, -1.2394e-01, -6.6772e-03, -5.1404e-02,  5.8547e-02,\n",
      "        -1.2281e-01,  1.3158e-03, -7.5456e-02, -6.5123e-02, -8.7693e-02,\n",
      "         6.5320e-02, -2.1265e-02, -3.7285e-02,  4.1005e-02,  4.9695e-02,\n",
      "        -6.7281e-02, -5.2154e-02, -7.8195e-03,  4.1187e-02,  4.5314e-02,\n",
      "        -2.2307e-02,  1.8562e-02, -3.1624e-02,  1.0709e-01,  7.9838e-02,\n",
      "         4.5924e-03, -3.6969e-02, -1.9349e-02, -5.1242e-02, -1.1036e-01,\n",
      "        -1.7156e-02, -3.9611e-03, -7.0930e-02, -6.2138e-02, -5.4107e-02,\n",
      "        -4.6743e-02, -6.2626e-02, -1.6184e-02, -3.8844e-02,  4.3067e-02,\n",
      "        -4.5245e-02, -9.5339e-02, -1.1333e-02, -9.4925e-02, -6.0949e-02,\n",
      "         1.3403e-02, -5.7001e-02, -1.7882e-04, -3.1056e-02,  3.6485e-02,\n",
      "        -2.4844e-02, -3.8216e-02,  2.7184e-02,  4.8756e-02, -8.0770e-02,\n",
      "         1.3767e-02, -1.1591e-01,  6.0599e-02, -9.2913e-02, -2.6273e-02,\n",
      "         3.6797e-02, -3.4717e-02, -6.5140e-02, -4.9010e-02, -4.9764e-03,\n",
      "        -1.1485e-02,  1.7246e-02,  2.6541e-04,  1.1249e-02, -4.8551e-02,\n",
      "         2.4803e-03,  3.8150e-03,  9.4858e-02,  5.3446e-02, -7.0102e-02,\n",
      "        -3.7033e-02, -4.0640e-03,  2.7160e-03,  2.3367e-02, -2.6884e-02,\n",
      "         2.0410e-02,  1.0467e-01,  6.5575e-03, -3.4802e-02,  3.4969e-03,\n",
      "        -1.1258e-01,  6.0316e-02,  6.0785e-02, -5.0316e-03,  3.8125e-02,\n",
      "        -3.8242e-02,  1.5232e-02,  2.4877e-02,  5.5707e-02, -1.7333e-02,\n",
      "        -8.3870e-03, -2.7598e-03,  6.0175e-03, -8.7342e-03,  2.7202e-03,\n",
      "         7.8466e-03,  4.3729e-02,  1.9243e-02,  2.9941e-02, -1.4906e-02,\n",
      "         5.1155e-02,  2.3867e-02,  2.0113e-03,  5.0543e-02,  4.8676e-02,\n",
      "        -2.8159e-02,  1.3715e-02, -9.4411e-02,  2.4325e-02,  5.0442e-02,\n",
      "         2.1608e-02,  3.7519e-02, -4.3826e-02,  1.7281e-02,  2.8674e-02,\n",
      "         3.0670e-02,  2.0565e-03,  5.0631e-02,  1.9613e-02, -3.5243e-02,\n",
      "        -5.1875e-02, -1.7375e-02, -8.8758e-02, -6.3684e-02, -4.2218e-02,\n",
      "         3.5579e-02,  7.8454e-02,  3.1185e-03,  6.4954e-02,  4.9188e-02,\n",
      "         4.8632e-02,  1.7933e-02, -1.9186e-03,  1.9123e-02,  9.3699e-03,\n",
      "        -9.2002e-03,  7.0588e-02,  7.5661e-03,  4.1431e-04, -3.7836e-02,\n",
      "        -1.7376e-02,  1.1259e-02,  1.3143e-02, -4.8586e-02,  4.6131e-02,\n",
      "        -1.7229e-03,  2.1808e-02,  1.0564e-02, -1.0189e-02, -9.3827e-02,\n",
      "        -3.6183e-02, -4.1041e-02,  1.6731e-02,  3.9917e-03,  7.5638e-02,\n",
      "        -3.1360e-02, -5.5899e-03,  3.9499e-02,  4.4890e-02, -2.6798e-02,\n",
      "         4.2297e-02, -9.0238e-02,  5.3642e-02, -6.6291e-02, -7.6680e-03,\n",
      "         4.9789e-02, -1.5843e-02, -5.7261e-03,  4.4548e-02,  2.5248e-02,\n",
      "        -1.4454e-02, -4.5186e-02, -1.1623e-02,  4.0611e-02,  1.4449e-01,\n",
      "        -1.4598e-02,  1.1984e-01, -5.3785e-02,  2.5975e-02, -4.6313e-02,\n",
      "         2.4377e-02, -4.1151e-02, -1.5798e-02,  5.4419e-02,  3.7001e-02,\n",
      "        -6.2048e-02,  8.5935e-03, -5.1462e-02, -1.6607e-02, -6.5805e-02,\n",
      "        -3.5540e-03,  5.9318e-02, -1.4863e-02,  2.5193e-02,  1.6731e-02,\n",
      "         4.2288e-02,  7.2243e-02,  2.8699e-02,  5.4654e-02, -7.8898e-02,\n",
      "         9.7674e-02, -1.2014e-02,  2.5968e-02, -3.5290e-02, -6.3095e-03,\n",
      "        -1.4691e-02, -1.8277e-02, -2.1165e-02, -9.4305e-03, -2.9362e-02,\n",
      "         1.0444e-01, -7.0496e-03, -4.3564e-02, -3.1179e-02,  2.5100e-02,\n",
      "        -2.2599e-02, -6.4648e-02, -2.3148e-01, -3.1508e-02, -1.4009e-01,\n",
      "         4.1234e-02,  5.6551e-02,  3.0445e-02, -6.5134e-03,  2.9356e-02,\n",
      "        -2.3961e-02, -2.0961e-02, -2.8567e-02, -1.5913e-02,  4.2031e-02,\n",
      "         4.1748e-04, -6.7495e-02, -4.1903e-02, -3.0430e-02,  3.4230e-02,\n",
      "         7.1165e-03, -5.4632e-02,  4.9230e-02,  7.9695e-02,  4.1417e-02,\n",
      "         1.1050e-01, -6.9364e-02,  1.6503e-03,  6.3633e-03, -3.1493e-02,\n",
      "        -4.0173e-02,  1.0574e-01, -2.7471e-02, -1.4568e-02, -5.7395e-03,\n",
      "         7.8667e-02,  3.1049e-02,  5.3252e-02, -8.5616e-02, -3.0315e-02,\n",
      "        -3.1951e-04,  3.9758e-02, -1.6567e-02,  3.3902e-02, -8.6822e-03,\n",
      "         9.7098e-02,  2.5617e-02,  5.4812e-03,  5.4592e-02,  1.3730e-02,\n",
      "        -7.2699e-02,  1.5442e-01,  1.1786e-02,  7.8614e-03, -8.3476e-02,\n",
      "         1.6274e-01, -2.6441e-02,  2.8781e-02, -6.1857e-04,  1.0527e-01,\n",
      "        -4.5044e-02, -4.9124e-02, -2.0630e-02,  4.0505e-02, -3.4631e-02,\n",
      "        -1.1339e-02, -5.9586e-02,  4.8904e-02, -4.8407e-03,  7.6267e-03,\n",
      "        -1.3286e-02,  7.1889e-02, -2.7649e-03, -4.1559e-02, -6.1008e-02,\n",
      "         1.9623e-02, -1.1826e-02,  9.7308e-03, -8.0617e-02,  4.3865e-02,\n",
      "        -7.9560e-02,  1.9731e-02,  2.1468e-02,  2.9930e-02,  3.5193e-02,\n",
      "         2.3778e-02, -9.0055e-02,  6.2192e-03, -3.7706e-02, -2.6777e-02,\n",
      "        -3.7376e-02, -1.3909e-02,  2.6083e-02, -1.0596e-02, -4.9610e-02,\n",
      "         5.7279e-02, -8.6085e-02,  5.4620e-02,  3.0482e-02,  2.6259e-02,\n",
      "         1.2022e-02, -1.8886e-02,  1.2601e-02,  3.4561e-02, -2.2214e-02,\n",
      "         1.6172e-02, -4.7363e-03,  2.2212e-02, -9.4738e-03,  1.1208e-01,\n",
      "        -1.9696e-02,  1.9606e-02,  1.8621e-02, -4.6987e-03,  3.6276e-02,\n",
      "         3.7455e-02, -5.1649e-02,  2.9051e-03, -7.4111e-02,  3.2023e-02,\n",
      "        -1.3625e-02,  2.0720e-02, -2.7129e-02, -5.3192e-03, -6.0962e-02,\n",
      "        -3.4979e-02,  1.3500e-04,  3.3182e-02, -1.2809e-01,  5.9894e-02,\n",
      "        -1.1375e-01,  1.3656e-02,  6.5875e-04,  2.1710e-02, -6.5095e-02,\n",
      "         7.0503e-02, -7.0486e-02,  4.2965e-02, -2.9503e-02,  6.5462e-02,\n",
      "        -5.4770e-02,  5.3502e-02, -1.9125e-02,  1.7265e-02,  2.3401e-02,\n",
      "        -3.1494e-02,  6.7667e-03,  3.7865e-02, -1.9995e-02,  2.5038e-02,\n",
      "        -4.6322e-02,  4.7339e-02, -2.1418e-02,  3.0150e-02,  6.5888e-02,\n",
      "         1.1421e-02,  1.1144e-01, -6.7627e-03, -5.6841e-02,  2.2283e-02,\n",
      "        -6.6946e-02, -3.6167e-02, -2.6622e-02,  4.4737e-02])), ('bert.encoder.layer.2.attention.output.LayerNorm.weight', tensor([0.6752, 0.5575, 0.6071, 0.4821, 0.5982, 0.6452, 0.5730, 0.5077, 0.6275,\n",
      "        0.5119, 0.6013, 0.3866, 0.4565, 0.5928, 0.5750, 0.6120, 0.6547, 0.5297,\n",
      "        0.4788, 0.6175, 0.4224, 0.4415, 0.6525, 0.4540, 0.2301, 0.2975, 0.2937,\n",
      "        0.4129, 0.6064, 0.5722, 0.7140, 0.6859, 0.5314, 0.5901, 0.6073, 0.5979,\n",
      "        0.4827, 0.4717, 0.3848, 0.5119, 0.4763, 0.5887, 0.6389, 0.5725, 0.5754,\n",
      "        0.6503, 0.6560, 0.7046, 0.7303, 0.7464, 0.7569, 0.6522, 0.6960, 0.7721,\n",
      "        0.7374, 0.7885, 0.7523, 0.7068, 0.7876, 0.8226, 0.7994, 0.8571, 0.8030,\n",
      "        0.8439, 0.8609, 0.9795, 1.0213, 0.9346, 0.9410, 0.8776, 0.9472, 0.9550,\n",
      "        0.9079, 0.9679, 0.9165, 1.0002, 0.9331, 1.0297, 0.9518, 1.1208, 1.0730,\n",
      "        0.8327, 0.9597, 1.0124, 1.0836, 0.9553, 1.0318, 1.0782, 0.9717, 1.0036,\n",
      "        1.0408, 1.0328, 1.0423, 0.9527, 1.0412, 0.9761, 1.0082, 1.0191, 1.0272,\n",
      "        1.0704, 1.0603, 0.7998, 1.0511, 1.0874, 1.0581, 0.7366, 1.0617, 0.9876,\n",
      "        1.0785, 0.9921, 0.9795, 1.0603, 1.1000, 1.0876, 1.0991, 1.0507, 1.0346,\n",
      "        1.0683, 1.0440, 0.8737, 1.3980, 1.0796, 1.0233, 1.0513, 1.0793, 1.0760,\n",
      "        1.0825, 0.9120, 0.9951, 1.0810, 0.8656, 1.0845, 0.9709, 1.0676, 1.1020,\n",
      "        1.0448, 1.0666, 1.0295, 1.0346, 1.0575, 1.0564, 1.0572, 1.0977, 1.1046,\n",
      "        1.0794, 1.0918, 1.0135, 0.9849, 1.0164, 1.0580, 1.1002, 1.0631, 1.1050,\n",
      "        1.0259, 0.9075, 1.0437, 0.9228, 0.9289, 1.0047, 0.8551, 1.1091, 1.1012,\n",
      "        1.0242, 1.1082, 1.0374, 0.9685, 1.0935, 0.9667, 0.9323, 1.1092, 1.0137,\n",
      "        1.0093, 1.0402, 1.0668, 1.1016, 1.0250, 1.0931, 1.0568, 1.1015, 1.0209,\n",
      "        1.0160, 0.8937, 1.0666, 1.0529, 0.9984, 1.0877, 1.0821, 1.1960, 1.1141,\n",
      "        1.0250, 1.0285, 1.0464, 1.0434, 1.0071, 1.0627, 0.9899, 1.0814, 1.0573,\n",
      "        1.0721, 0.9760, 1.0765, 1.0891, 1.0061, 1.0565, 1.0891, 1.0461, 0.9549,\n",
      "        1.0337, 1.1149, 0.9975, 1.0341, 1.0678, 1.0763, 1.0475, 0.9798, 1.0801,\n",
      "        1.0633, 1.0436, 0.9157, 1.0299, 1.1141, 1.0460, 0.9473, 1.0079, 1.0465,\n",
      "        1.0636, 1.0119, 1.1315, 1.0905, 1.0257, 1.1578, 1.0831, 1.0650, 1.0295,\n",
      "        1.0217, 1.0840, 0.8556, 1.0851, 0.9869, 1.0394, 0.9894, 1.0379, 1.0583,\n",
      "        1.0634, 1.0756, 1.0509, 1.0160, 0.9221, 1.2241, 1.0509, 1.0401, 1.0358,\n",
      "        1.1167, 1.1458, 1.0561, 1.0860, 1.0873, 1.0641, 0.9451, 1.0631, 1.0331,\n",
      "        1.0712, 1.1140, 1.0920, 1.0249, 1.1189, 1.0623, 1.0955, 0.9755, 1.0056,\n",
      "        1.1114, 1.0696, 1.0737, 1.0319, 0.9928, 1.1190, 1.1427, 0.9854, 1.0995,\n",
      "        0.9870, 1.1201, 1.1162, 1.1439, 0.9211, 1.1091, 1.0919, 1.1183, 0.8817,\n",
      "        1.0357, 1.0659, 1.0479, 1.0276, 1.0723, 1.0637, 0.9789, 1.1247, 1.1375,\n",
      "        1.0695, 1.1204, 1.0610, 1.0670, 0.9785, 1.0796, 1.0664, 1.0301, 1.0696,\n",
      "        1.0662, 1.0150, 1.0616, 0.9853, 0.9826, 1.0154, 1.0791, 1.0262, 1.0328,\n",
      "        1.0438, 1.0453, 1.1220, 1.0224, 1.1012, 1.0174, 1.0462, 1.0378, 1.0465,\n",
      "        1.0772, 1.1773, 1.0322, 1.0649, 1.0640, 0.9606, 1.0473, 1.0304, 1.0725,\n",
      "        0.8910, 0.9087, 1.1616, 1.0542, 0.9621, 1.0967, 1.0772, 1.1364, 1.0285,\n",
      "        1.0393, 1.0720, 1.0269, 1.0705, 1.0471, 0.9947, 1.1760, 1.0827, 1.0619,\n",
      "        1.0333, 1.0240, 0.9549, 1.0341, 1.1496, 1.1054, 1.0033, 1.0755, 1.0724,\n",
      "        0.9714, 0.9871, 1.0756, 0.7863, 0.9946, 1.0214, 1.0390, 0.9997, 1.0547,\n",
      "        0.8713, 1.0694, 0.9997, 1.0818, 1.0637, 1.0878, 1.0028, 0.9886, 0.9816,\n",
      "        1.0610, 1.0145, 1.0058, 1.0493, 1.0753, 1.1498])), ('bert.encoder.layer.2.attention.output.LayerNorm.bias', tensor([-1.3259e-01, -1.8079e-01, -1.0474e-01, -7.9583e-02,  4.6936e-02,\n",
      "        -1.2944e-01,  1.2367e-01, -1.3933e-01, -2.4055e-01, -6.5854e-02,\n",
      "         6.1762e-02, -8.8105e-02, -3.8478e-02, -2.6960e-01, -3.0903e-02,\n",
      "        -5.4173e-02, -1.8933e-01, -4.4814e-02,  2.0514e-02, -3.1140e-04,\n",
      "        -1.1121e-01,  6.6468e-02, -1.1665e-01,  2.4701e-01,  1.5171e-01,\n",
      "        -7.7718e-02, -8.6513e-02,  6.8114e-02, -1.4185e-01, -1.3746e-01,\n",
      "        -1.1461e-01, -1.3469e-01, -1.3399e-01, -1.0085e-01, -2.1250e-01,\n",
      "        -1.5173e-02, -8.1136e-03, -7.9218e-02, -1.0811e-01, -1.0273e-02,\n",
      "        -1.6621e-01, -2.3287e-01, -1.7731e-01, -1.7959e-01, -2.8737e-02,\n",
      "        -1.3589e-01, -1.1957e-01,  8.3910e-02, -2.1947e-01,  7.7748e-02,\n",
      "         4.5022e-03,  8.4168e-03,  1.3696e-01,  1.5634e-01, -1.5865e-01,\n",
      "        -4.5442e-02, -2.8702e-01,  3.8403e-02, -1.7888e-01, -2.0984e-01,\n",
      "        -5.6711e-02, -3.1428e-02, -4.0149e-02, -4.4146e-02,  1.0330e-02,\n",
      "        -8.0540e-02,  1.0526e-01, -3.8993e-02,  2.0847e-01,  2.2166e-02,\n",
      "         3.9418e-02,  1.0247e-01,  6.7808e-02,  1.5112e-01, -2.8377e-01,\n",
      "         9.7151e-02,  6.9937e-02, -8.4888e-02, -7.4357e-02, -5.2263e-02,\n",
      "         1.6895e-01,  2.7804e-01,  2.1515e-01, -1.0887e-01, -5.5948e-02,\n",
      "        -1.7842e-01,  8.2587e-02,  2.7156e-01, -1.7828e-01,  3.1399e-01,\n",
      "         5.7987e-03, -1.1913e-01, -6.9570e-03,  1.4814e-01,  6.8309e-02,\n",
      "         1.6805e-01, -2.0342e-02,  6.9426e-02, -1.0206e-01,  1.0265e-01,\n",
      "        -1.1735e-01,  2.7843e-01, -2.6075e-02,  5.0354e-02, -1.0309e-01,\n",
      "         4.1068e-02, -9.2683e-02,  1.5606e-01,  2.0642e-02,  2.0126e-01,\n",
      "         4.3917e-02,  1.5258e-01, -2.3165e-01,  1.8851e-01,  1.0243e-01,\n",
      "         4.7801e-02,  1.1280e-01, -1.4049e-01,  8.2386e-02, -4.2920e-02,\n",
      "         1.4924e-01,  1.6324e-01,  8.5029e-02,  4.2608e-03,  1.1038e-01,\n",
      "        -1.3286e-01, -7.1202e-03, -1.5167e-02, -1.3758e-01,  1.6542e-01,\n",
      "         1.2388e-01,  2.8024e-01, -5.2586e-02, -6.5840e-02,  7.7221e-02,\n",
      "         7.6432e-02,  7.7545e-02,  3.8214e-03, -1.2311e-01, -9.3295e-02,\n",
      "        -5.8842e-02,  2.1482e-01, -4.1367e-02, -1.4189e-04, -6.6233e-02,\n",
      "        -3.1558e-02, -1.7901e-01, -5.7333e-02, -1.6991e-02,  1.3714e-01,\n",
      "         2.3552e-02,  7.5379e-02, -2.2222e-02,  3.3927e-02, -2.9118e-02,\n",
      "        -3.0002e-02, -5.5061e-02,  9.6705e-02, -3.1058e-02,  2.8413e-01,\n",
      "        -8.1442e-02, -2.5767e-02, -7.0662e-02,  2.1381e-01, -5.9843e-02,\n",
      "         3.3699e-01, -1.3047e-01,  2.0784e-01, -3.0942e-02,  1.1890e-01,\n",
      "         4.4626e-02, -1.5697e-01, -1.0284e-01,  1.9349e-01,  6.1592e-03,\n",
      "         5.7304e-02, -1.5096e-02,  8.8214e-02, -4.1864e-02,  3.4539e-01,\n",
      "        -6.8274e-02,  1.6177e-01, -3.8417e-02,  9.3441e-02, -2.4017e-01,\n",
      "        -1.1998e-01, -1.3768e-01,  3.9274e-01,  1.5027e-01,  1.7104e-01,\n",
      "        -1.3182e-01,  1.3732e-01, -9.6927e-02,  3.7987e-02, -2.9387e-01,\n",
      "         4.7980e-02,  1.7546e-01,  7.7417e-02, -7.4253e-02,  1.0690e-02,\n",
      "        -1.9417e-01,  6.6269e-02, -1.0063e-01,  1.2993e-01, -1.9030e-01,\n",
      "         1.1665e-01,  1.5844e-02,  2.2945e-01, -5.0134e-02,  1.2609e-01,\n",
      "        -6.4630e-02,  2.3029e-02, -7.4092e-02,  8.9393e-02, -7.9488e-02,\n",
      "         1.0116e-01, -1.0529e-01, -4.1865e-02, -1.6288e-01,  4.4072e-02,\n",
      "        -2.3399e-01, -1.5840e-01, -4.5204e-01, -8.3833e-02, -3.6490e-01,\n",
      "         1.9805e-02,  6.5522e-02,  2.1370e-01,  1.0002e-01,  7.6903e-02,\n",
      "        -9.7213e-02,  1.2340e-02, -2.3944e-01,  3.9757e-02, -2.7690e-02,\n",
      "         3.4942e-02, -5.9526e-02, -5.8556e-02, -1.8008e-01,  4.0034e-02,\n",
      "        -1.4798e-01,  1.1147e-01, -4.4269e-02,  1.1279e-01,  6.5373e-02,\n",
      "         3.1279e-01, -3.0363e-01, -2.6234e-02, -1.4082e-01,  4.9463e-02,\n",
      "        -1.4121e-03,  2.7379e-01, -1.5329e-01, -2.0929e-01,  1.5955e-02,\n",
      "         1.3018e-01,  7.6129e-02, -2.3936e-02, -3.4056e-01, -6.6264e-02,\n",
      "         3.3479e-02, -4.0634e-02,  8.7475e-03,  8.8088e-02,  1.7903e-02,\n",
      "         2.2140e-01,  3.0009e-04,  1.4879e-01, -5.4741e-02, -3.2637e-02,\n",
      "        -1.2035e-01,  3.5923e-01, -2.1300e-01,  6.1123e-02, -8.6584e-02,\n",
      "         3.3854e-01, -1.7930e-01,  1.5283e-01, -6.4919e-02,  2.9309e-01,\n",
      "        -2.1941e-01, -7.4164e-02, -8.1210e-02, -1.2520e-01, -1.1224e-01,\n",
      "         1.9193e-01, -1.8849e-01,  2.7933e-01, -1.1534e-01, -1.0039e-01,\n",
      "        -1.1573e-01,  1.7122e-01, -2.1965e-01, -8.3788e-02, -9.0997e-02,\n",
      "         2.0258e-01, -8.9211e-02,  1.1731e-01, -2.7652e-01,  3.1180e-01,\n",
      "        -1.9963e-01, -1.5172e-01,  6.0471e-02,  1.0376e-02, -5.0230e-02,\n",
      "         5.1814e-02, -3.1191e-01,  2.5434e-01, -2.6665e-02,  1.3041e-01,\n",
      "        -1.2942e-01,  2.2246e-02,  1.1422e-01, -1.6218e-02, -1.2336e-01,\n",
      "         1.3746e-01, -1.9162e-01,  1.3648e-01,  1.0030e-01, -4.3592e-02,\n",
      "         1.1076e-01, -7.0101e-02, -7.5466e-02,  1.4708e-01, -2.7540e-02,\n",
      "         1.1823e-01, -2.1139e-01, -1.3736e-01,  1.6559e-01,  2.0119e-01,\n",
      "        -1.7067e-01,  2.1312e-01, -1.6363e-01,  2.4182e-01, -7.9268e-02,\n",
      "         2.3061e-01, -1.0762e-01,  6.2506e-02,  2.3344e-02,  1.0672e-01,\n",
      "        -8.2161e-02,  6.4784e-02, -5.0717e-02,  1.1032e-01, -2.6658e-01,\n",
      "        -7.0885e-03, -7.6914e-02, -6.9459e-02, -4.6170e-01,  2.3237e-01,\n",
      "        -3.0476e-01,  1.0767e-01, -1.6421e-01,  2.8843e-02, -1.3695e-02,\n",
      "         1.7190e-01, -1.2217e-01,  1.5490e-01, -3.6674e-02,  2.8160e-01,\n",
      "        -2.1694e-01,  4.5198e-02, -4.9525e-02,  1.0499e-01, -2.7520e-03,\n",
      "        -2.1541e-01,  7.4664e-02,  1.7386e-01, -1.5460e-01,  1.5595e-01,\n",
      "        -2.4224e-01, -2.8298e-02, -5.2357e-02, -8.9142e-03,  7.9491e-02,\n",
      "        -1.8061e-03,  1.5830e-01, -4.4815e-02, -1.6468e-01,  2.0345e-01,\n",
      "         1.3903e-02, -1.5968e-01, -2.1655e-01,  1.6069e-01])), ('bert.encoder.layer.2.intermediate.dense.weight', tensor([[ 0.0335,  0.0464,  0.0246,  ..., -0.0357,  0.0373, -0.1287],\n",
      "        [ 0.0027, -0.0056,  0.0055,  ..., -0.0418, -0.0387, -0.0154],\n",
      "        [ 0.0347,  0.0162,  0.0294,  ...,  0.0543,  0.0461,  0.0671],\n",
      "        ...,\n",
      "        [-0.0020,  0.0315, -0.0103,  ...,  0.0737,  0.1322,  0.0368],\n",
      "        [ 0.0092, -0.0275,  0.0090,  ...,  0.0621,  0.0068,  0.0278],\n",
      "        [ 0.0816, -0.0803, -0.0813,  ...,  0.0503, -0.0598,  0.0387]])), ('bert.encoder.layer.2.intermediate.dense.bias', tensor([-0.0763, -0.0542, -0.0942,  ..., -0.0835, -0.0601, -0.0856])), ('bert.encoder.layer.2.output.dense.weight', tensor([[-0.0780, -0.0116,  0.0288,  ...,  0.0019, -0.0995,  0.1731],\n",
      "        [ 0.0092,  0.0075,  0.1024,  ...,  0.0463, -0.0357, -0.0093],\n",
      "        [-0.0540,  0.0325, -0.0022,  ..., -0.0244,  0.0493,  0.0458],\n",
      "        ...,\n",
      "        [-0.0466,  0.0597, -0.0544,  ...,  0.0453,  0.0244,  0.0023],\n",
      "        [-0.0482,  0.0357,  0.0408,  ...,  0.1108, -0.0206,  0.0961],\n",
      "        [ 0.0651,  0.0667,  0.0569,  ...,  0.0494, -0.0506, -0.0631]])), ('bert.encoder.layer.2.output.dense.bias', tensor([ 9.0787e-03,  5.1186e-02, -1.9129e-02,  8.9623e-02, -5.2677e-02,\n",
      "        -2.0490e-02,  9.4741e-03, -4.8518e-02, -5.1277e-02,  2.0281e-02,\n",
      "         6.3679e-02, -1.8805e-02, -8.6435e-02,  1.0332e-01, -1.7661e-02,\n",
      "         2.9412e-02,  5.4855e-02, -5.1076e-02, -8.6519e-03, -5.6275e-02,\n",
      "         2.4511e-02,  1.5012e-01,  2.4453e-02, -2.3074e-02,  3.5992e-02,\n",
      "         6.7159e-02,  2.8427e-03, -2.9647e-02, -2.4084e-02,  7.1984e-02,\n",
      "        -3.9543e-03, -1.2974e-02, -4.5240e-03, -6.1782e-02, -9.3545e-02,\n",
      "        -1.6276e-02, -1.7509e-02,  8.0794e-02, -2.3033e-02, -6.6867e-02,\n",
      "        -5.3657e-02, -4.1179e-02, -4.5702e-02,  2.6819e-02, -1.3732e-01,\n",
      "         3.0717e-02, -9.6846e-03, -5.9879e-02, -6.7756e-03,  4.8199e-02,\n",
      "         9.3614e-02, -7.2253e-02,  1.5272e-02,  9.7075e-03, -4.0329e-02,\n",
      "         8.9801e-02, -4.9665e-03, -1.1284e-02,  1.1037e-02,  1.7554e-02,\n",
      "         1.3897e-02, -1.1916e-02, -3.3035e-02, -1.3946e-02,  3.9858e-02,\n",
      "        -3.6053e-02, -2.7036e-02, -7.6749e-03,  5.8746e-03, -1.5649e-02,\n",
      "         2.9884e-02,  4.7660e-02, -1.0713e-02, -4.9771e-03,  1.0759e-01,\n",
      "         7.7707e-03,  1.7875e-02, -1.4279e-02,  2.0741e-02, -3.6386e-02,\n",
      "        -4.3827e-02, -9.0410e-03,  1.5102e-02, -3.9005e-03,  3.8030e-02,\n",
      "        -2.5834e-02,  9.4904e-03,  3.2945e-02,  2.0621e-02,  4.7967e-02,\n",
      "         2.2841e-02, -3.3979e-02, -4.2803e-02,  3.1416e-03,  3.0531e-02,\n",
      "        -8.5493e-02,  1.3355e-02, -2.7200e-02, -1.4646e-02, -4.0967e-02,\n",
      "         6.3234e-02,  3.8286e-02,  1.3489e-02, -1.9862e-02,  1.8700e-02,\n",
      "        -8.8265e-03,  2.3679e-02,  1.8551e-02, -2.2098e-02, -2.3890e-03,\n",
      "         3.8364e-02, -1.7607e-02, -1.5326e-03,  1.3750e-02,  2.2000e-03,\n",
      "        -4.4685e-02,  3.1905e-02,  2.3586e-02, -4.5002e-04,  8.3989e-03,\n",
      "        -1.8226e-02, -2.2028e-02,  3.1421e-02,  3.2678e-02, -6.7706e-03,\n",
      "         2.1510e-02, -1.6873e-02, -4.5575e-02,  5.2771e-02, -4.4592e-03,\n",
      "        -5.4207e-03, -4.2357e-03,  3.6184e-02,  1.3080e-02,  7.3020e-03,\n",
      "        -1.0501e-02,  1.1825e-02, -2.2041e-02,  1.1122e-02,  8.8519e-03,\n",
      "        -1.5302e-02,  3.6767e-02, -8.4184e-03, -8.7127e-03,  1.3380e-02,\n",
      "        -1.4527e-02,  3.8435e-03, -8.6526e-03, -4.3298e-02,  7.4426e-04,\n",
      "        -7.0309e-03,  2.1745e-02,  3.1110e-02, -2.8469e-02,  9.1733e-03,\n",
      "        -7.7655e-02, -2.4230e-02,  3.6376e-02,  2.3343e-03, -6.6918e-02,\n",
      "        -1.5033e-02, -7.0315e-03,  5.1017e-02,  2.3406e-02, -4.9992e-02,\n",
      "        -3.1073e-02,  3.8787e-02, -2.3565e-02, -1.8081e-02,  4.3758e-02,\n",
      "         5.2132e-02,  1.9732e-02,  4.3678e-03, -4.1456e-02, -5.6426e-03,\n",
      "        -2.2554e-02, -5.4400e-02,  1.2325e-02, -1.5370e-02,  3.1670e-03,\n",
      "         2.5274e-02,  5.7240e-02, -4.2444e-02,  2.8793e-02, -3.0716e-02,\n",
      "         8.6998e-03,  1.9991e-02, -1.7270e-02,  4.2144e-03, -1.8027e-02,\n",
      "        -5.4677e-02, -1.6458e-02, -6.5447e-02, -1.1926e-02, -6.7871e-02,\n",
      "        -2.5951e-02, -5.3239e-03, -1.0922e-02, -3.3178e-02, -6.9164e-02,\n",
      "         2.4746e-02, -6.2030e-03,  4.1504e-02,  1.7862e-02, -2.1867e-02,\n",
      "         5.6153e-03, -1.4586e-03,  3.2578e-02,  1.1417e-02, -7.9715e-03,\n",
      "        -2.6832e-02, -2.3565e-03,  2.7994e-02,  4.7394e-03,  1.0638e-02,\n",
      "         4.7302e-02, -1.0174e-04, -7.0877e-03,  8.9498e-03,  2.1456e-02,\n",
      "        -4.6324e-03, -2.0136e-02, -1.6236e-01, -9.5555e-03, -3.2513e-02,\n",
      "        -2.2113e-02,  3.9843e-02, -2.1002e-02,  1.9487e-02,  6.7028e-03,\n",
      "         6.2603e-02, -2.2841e-02, -1.7678e-02,  7.0921e-03, -1.4896e-02,\n",
      "        -1.4890e-04,  1.1394e-02, -8.0587e-03,  1.2556e-02,  1.5422e-02,\n",
      "         4.6573e-02,  1.4943e-02,  1.8251e-02,  1.2140e-02,  2.1568e-02,\n",
      "        -5.3568e-02,  6.2634e-02, -6.4664e-02, -5.0545e-02, -1.7467e-02,\n",
      "        -7.7432e-03, -1.8870e-02,  1.7989e-02,  3.4928e-02, -3.5011e-02,\n",
      "        -3.4430e-03,  3.7760e-02, -1.6061e-02, -9.1651e-02, -2.2080e-02,\n",
      "         5.3762e-02,  5.0622e-02,  8.1169e-03,  1.0788e-02, -2.7195e-02,\n",
      "        -2.2483e-02,  4.7903e-03, -2.2646e-02,  6.2020e-02,  3.4063e-02,\n",
      "        -4.1963e-02,  6.9428e-02, -1.2582e-02,  6.2649e-02, -4.6647e-02,\n",
      "         8.3755e-02,  9.1826e-03, -2.0287e-02,  1.0239e-02, -8.0997e-03,\n",
      "         1.4854e-02,  1.7349e-02,  6.6943e-02,  2.2286e-02,  3.0339e-02,\n",
      "        -5.0280e-02,  7.0927e-03,  1.0233e-02, -2.7002e-03, -6.4417e-02,\n",
      "        -4.5938e-02,  4.5844e-03,  1.7382e-02, -1.6578e-02, -8.6701e-03,\n",
      "        -5.7598e-02,  3.0886e-02, -4.2156e-03, -3.2307e-02, -5.1153e-03,\n",
      "         3.0250e-03, -1.7392e-02,  3.0807e-02, -6.3936e-03,  3.0699e-02,\n",
      "         2.5770e-02,  4.7964e-02,  9.0192e-03,  1.7405e-02, -6.9402e-02,\n",
      "         1.0735e-02,  3.5024e-02, -3.3785e-02,  2.5464e-02, -9.5090e-03,\n",
      "        -4.7927e-03,  1.0340e-02,  5.9412e-02,  1.5383e-02,  8.7573e-03,\n",
      "         1.8066e-02,  1.1828e-02, -2.9487e-03,  5.3862e-02,  2.1886e-02,\n",
      "        -2.3801e-02,  1.9076e-02, -1.8980e-02,  1.4248e-02,  7.5513e-03,\n",
      "        -2.3119e-02, -1.0110e-02, -1.1298e-02, -4.2925e-02, -4.2258e-02,\n",
      "         7.6529e-02,  8.8591e-03,  1.4072e-02, -2.6481e-02,  1.8971e-02,\n",
      "         2.3355e-02,  2.3339e-02,  4.2456e-02, -5.5725e-03, -8.3241e-03,\n",
      "        -5.0599e-02,  6.2460e-02, -4.2958e-02, -9.6352e-02,  2.3926e-02,\n",
      "        -3.3174e-02,  2.1227e-02, -2.6647e-02,  1.3368e-02, -4.0272e-02,\n",
      "         7.4157e-02, -2.6602e-02,  6.7831e-03, -7.5311e-02,  2.7400e-02,\n",
      "        -1.6347e-03,  2.4489e-02, -7.1381e-02, -3.7889e-02, -1.2378e-02,\n",
      "        -3.0762e-02, -1.4824e-03, -3.2419e-02, -8.3367e-03,  3.0509e-02,\n",
      "        -5.0178e-03, -1.7116e-02,  1.3319e-02, -2.7866e-02,  2.0794e-02,\n",
      "         3.7398e-02, -2.2346e-02, -5.8268e-02,  4.3051e-02, -2.3673e-02,\n",
      "        -2.2041e-03, -1.3136e-03, -3.0208e-02,  2.1707e-02])), ('bert.encoder.layer.2.output.LayerNorm.weight', tensor([0.7527, 0.7517, 0.7435, 0.7567, 0.7238, 0.7701, 0.7051, 0.6909, 0.7338,\n",
      "        0.7296, 0.7017, 0.7095, 0.6840, 0.7160, 0.7276, 0.7223, 0.7542, 0.7093,\n",
      "        0.7130, 0.7751, 0.6920, 0.7056, 0.7685, 0.6729, 0.5714, 0.6681, 0.6328,\n",
      "        0.7013, 0.7507, 0.7001, 0.7452, 0.7588, 0.7050, 0.7393, 0.7388, 0.7808,\n",
      "        0.6873, 0.7189, 0.6830, 0.7271, 0.7121, 0.7415, 0.7237, 0.6650, 0.7519,\n",
      "        0.7685, 0.7190, 0.7877, 0.7398, 0.8334, 0.8230, 0.7898, 0.8401, 0.8299,\n",
      "        0.8093, 0.8282, 0.7626, 0.8187, 0.7828, 0.8308, 0.8289, 0.8739, 0.8775,\n",
      "        0.8589, 0.8924, 0.8651, 0.9181, 0.8815, 0.8702, 0.8722, 0.8687, 0.8808,\n",
      "        0.8589, 0.9162, 0.7420, 0.9189, 0.8755, 0.9515, 0.9040, 0.9427, 0.7786,\n",
      "        0.7864, 0.8931, 0.9178, 0.9289, 0.8698, 0.9197, 0.9029, 0.8898, 0.8071,\n",
      "        0.9285, 0.8862, 0.9430, 0.8952, 0.9261, 0.8786, 0.9472, 0.9275, 0.9452,\n",
      "        0.9242, 0.9471, 0.7618, 0.9379, 0.9334, 0.9054, 0.8138, 0.9300, 0.9030,\n",
      "        0.9963, 0.8931, 0.9395, 0.9121, 0.9256, 0.8983, 0.9468, 0.9426, 0.9183,\n",
      "        0.9165, 0.9434, 0.8943, 0.9903, 0.9705, 0.9506, 0.9207, 0.9353, 0.8794,\n",
      "        0.9459, 0.8659, 0.8793, 0.9081, 0.7896, 0.8543, 0.9183, 0.9144, 0.9624,\n",
      "        0.9401, 0.9248, 0.9097, 0.9225, 0.9274, 0.9360, 0.8880, 0.9734, 0.9361,\n",
      "        0.9529, 0.9331, 0.8962, 0.9099, 0.9353, 0.9244, 0.9626, 0.9287, 0.9543,\n",
      "        0.9370, 0.8536, 0.9232, 0.8711, 0.8799, 0.9082, 0.7818, 0.9571, 0.9601,\n",
      "        0.9154, 0.9327, 0.9271, 0.8013, 0.9743, 0.8706, 0.9014, 0.9480, 0.9093,\n",
      "        0.8950, 0.9398, 0.9159, 0.9894, 0.9443, 0.9569, 0.9245, 0.9581, 0.8098,\n",
      "        0.9378, 0.8376, 0.9187, 0.9127, 0.8511, 0.9263, 0.9436, 1.0736, 0.9380,\n",
      "        0.9158, 0.9223, 0.9079, 0.9091, 0.9136, 0.8700, 0.8883, 0.9380, 0.9423,\n",
      "        0.9263, 0.9183, 0.9008, 0.9405, 0.9163, 0.9394, 0.8920, 0.9262, 0.9034,\n",
      "        0.8799, 0.9165, 0.9365, 0.9153, 0.9366, 0.9247, 0.9591, 0.9543, 0.9535,\n",
      "        0.9255, 0.9015, 0.8768, 0.9276, 0.8721, 0.9049, 0.7967, 0.9360, 0.8013,\n",
      "        0.9324, 0.9123, 0.8845, 0.9067, 0.9200, 0.9479, 0.9512, 0.8831, 0.9157,\n",
      "        0.9046, 0.9701, 0.8408, 0.9394, 0.8854, 0.9351, 0.9146, 0.9285, 0.9340,\n",
      "        0.9252, 0.9620, 0.8384, 0.8774, 0.8801, 1.0318, 0.9193, 0.9350, 0.8889,\n",
      "        0.9296, 0.8500, 0.9634, 0.9152, 0.9502, 0.9338, 0.8026, 0.9341, 0.9549,\n",
      "        0.9417, 0.9651, 0.9469, 0.9208, 0.9106, 0.9248, 0.9778, 0.8683, 0.9164,\n",
      "        0.9767, 0.8840, 0.9015, 0.9361, 0.8916, 0.8901, 0.9398, 0.8900, 0.9166,\n",
      "        0.8453, 0.9205, 0.9595, 0.9575, 0.9125, 0.9127, 0.9273, 0.9157, 0.8113,\n",
      "        0.9032, 0.9354, 0.9125, 0.9073, 0.9013, 0.9304, 0.9040, 0.9123, 0.9750,\n",
      "        0.9496, 0.8615, 0.8360, 0.8800, 0.8855, 0.9360, 0.9398, 0.9263, 0.9184,\n",
      "        0.9007, 0.8957, 0.9105, 0.8997, 0.8909, 0.9307, 0.9523, 0.9105, 0.8909,\n",
      "        0.9294, 0.9016, 0.9274, 0.9324, 0.9706, 0.9423, 0.9189, 0.9335, 0.8911,\n",
      "        0.9189, 0.9415, 0.9303, 0.9176, 0.9396, 0.8488, 0.9015, 0.8850, 0.9134,\n",
      "        0.8373, 0.8974, 0.9104, 0.9525, 0.8735, 0.9307, 0.9530, 0.9895, 0.9525,\n",
      "        0.9082, 0.9086, 0.9037, 0.9479, 0.9446, 0.9323, 0.8133, 0.8900, 0.8791,\n",
      "        0.9208, 0.9213, 0.8912, 0.9010, 0.9326, 0.9431, 0.8989, 0.9491, 0.8866,\n",
      "        0.8704, 0.9305, 0.9014, 0.8542, 0.8920, 0.8543, 0.9611, 0.8740, 0.9270,\n",
      "        0.8322, 0.8852, 0.9329, 0.9525, 0.9326, 0.9428, 0.8934, 0.8966, 0.9070,\n",
      "        0.8797, 0.8840, 0.8751, 0.9157, 0.9390, 0.9415])), ('bert.encoder.layer.2.output.LayerNorm.bias', tensor([ 5.9871e-02,  3.7807e-02,  2.0281e-02,  3.0172e-02, -5.2627e-02,\n",
      "         3.7187e-02,  2.8592e-02, -1.1329e-02, -5.1805e-02, -9.1474e-03,\n",
      "         8.6078e-02,  2.8256e-02, -7.6947e-02,  1.1351e-01, -7.3119e-03,\n",
      "         1.6389e-02,  9.2700e-02, -2.6540e-02,  1.8644e-02, -2.0934e-02,\n",
      "         5.1792e-02,  7.6174e-02,  1.1161e-01, -9.2597e-02, -3.1650e-03,\n",
      "         5.2065e-03,  1.2542e-02,  2.1829e-02,  5.4957e-02,  1.4902e-01,\n",
      "         9.5331e-02,  6.5995e-02,  6.0886e-02, -1.2580e-02, -9.6622e-03,\n",
      "         3.1214e-02, -1.4423e-02,  7.2362e-02,  1.5429e-02, -4.3455e-02,\n",
      "         2.4491e-02,  1.0001e-01,  5.1801e-02,  6.6178e-02, -9.9285e-02,\n",
      "         7.6982e-02,  1.1281e-01, -6.4785e-02,  7.6714e-02, -2.4266e-02,\n",
      "         5.1781e-02, -4.4494e-02, -7.6353e-02, -1.0916e-02,  3.6619e-02,\n",
      "         1.0696e-01,  1.0595e-01,  4.2196e-02,  6.7231e-02,  6.1473e-02,\n",
      "         1.3864e-02,  8.0465e-02, -6.3349e-02,  2.1964e-03,  4.3933e-02,\n",
      "         1.0485e-01, -4.2412e-02,  2.0787e-02, -9.8669e-02, -8.7425e-02,\n",
      "        -7.5108e-03,  2.7119e-02, -2.1413e-02, -2.2060e-02,  2.4418e-01,\n",
      "        -8.1288e-02, -5.7540e-02,  3.1957e-02,  3.6884e-02,  1.0377e-02,\n",
      "        -1.5134e-01, -1.6513e-01, -4.9951e-02, -2.2672e-02,  3.5936e-02,\n",
      "         2.1447e-02, -5.3090e-02, -1.1226e-01,  8.2806e-02, -1.3189e-01,\n",
      "         2.1177e-02,  5.1224e-02, -1.2612e-02, -8.1517e-02,  3.2980e-02,\n",
      "        -1.1705e-01,  1.0124e-02, -3.6981e-02,  4.5278e-02, -2.7002e-03,\n",
      "         7.3961e-02, -1.7515e-01,  2.9981e-02,  1.5064e-02,  5.9403e-02,\n",
      "        -7.0466e-02,  4.0800e-02, -4.6736e-02, -1.3939e-02, -3.0074e-02,\n",
      "        -2.9413e-03, -3.2816e-02,  7.7117e-02, -6.5578e-02, -8.6298e-02,\n",
      "         1.3209e-02, -3.7150e-02,  9.3824e-02,  1.6176e-02, -2.0214e-02,\n",
      "        -6.1882e-02, -5.0369e-02, -9.6041e-02, -5.5444e-02, -9.2296e-03,\n",
      "         1.2263e-01,  2.4172e-03, -5.0453e-02,  1.2050e-01,  1.9257e-02,\n",
      "        -1.1222e-01, -4.5334e-02,  4.6159e-02,  3.8699e-02,  1.6537e-02,\n",
      "        -7.1343e-03,  1.2858e-02, -1.1914e-02,  6.2835e-02, -2.6526e-02,\n",
      "         2.1524e-02, -3.9572e-02, -1.8743e-03, -2.2560e-02,  4.6010e-02,\n",
      "         3.4106e-03,  1.1467e-01,  8.1954e-02, -1.8952e-02, -5.7619e-02,\n",
      "         1.0365e-02, -3.6098e-02,  4.7551e-02, -3.0044e-03,  1.1270e-02,\n",
      "        -4.5580e-03,  3.9000e-02, -5.0525e-02, -3.8206e-03, -1.3002e-01,\n",
      "         4.4558e-02, -8.1090e-03,  9.1397e-02, -8.9268e-02,  1.7059e-02,\n",
      "        -1.8609e-01,  5.1370e-02, -1.4304e-01,  5.1985e-02, -1.1361e-02,\n",
      "        -3.2473e-02,  4.3010e-02,  5.2825e-02, -1.4130e-01,  7.3998e-04,\n",
      "         2.2729e-02,  1.5910e-02, -3.7558e-02, -1.4013e-02, -1.2692e-01,\n",
      "         3.3101e-02, -1.0223e-01, -2.1537e-03,  1.2281e-02,  9.4325e-02,\n",
      "         1.0036e-01,  8.7834e-02, -9.6192e-02, -4.3951e-02, -1.0902e-01,\n",
      "        -7.1457e-03, -3.2745e-02,  7.0132e-02, -4.6038e-02,  8.7194e-02,\n",
      "        -4.7406e-02, -2.8042e-02, -3.1532e-02, -6.0473e-03, -7.5090e-02,\n",
      "         6.7345e-02, -3.7776e-02,  1.0690e-01, -4.3686e-02,  1.3452e-01,\n",
      "         4.6154e-02,  9.5593e-03, -8.2342e-02,  1.2792e-01, -5.1449e-02,\n",
      "         1.9072e-02,  7.2021e-02,  4.1697e-02, -4.1199e-02,  1.0652e-02,\n",
      "        -3.6306e-03,  2.5694e-02,  1.2991e-03,  4.7550e-02, -9.9097e-03,\n",
      "         1.6293e-01,  5.0303e-02,  4.4435e-02,  1.0889e-02,  1.3137e-01,\n",
      "        -8.5826e-02, -1.0123e-02, -1.3081e-01,  9.4932e-03,  3.2692e-03,\n",
      "         1.1918e-01, -6.9816e-04,  1.2664e-01, -2.3175e-02,  3.0366e-02,\n",
      "        -3.2709e-02,  6.3371e-02,  1.0400e-02,  8.5877e-02, -4.0142e-02,\n",
      "         1.0612e-01, -1.0039e-02,  1.3938e-02, -4.1157e-02, -5.5017e-02,\n",
      "        -9.4928e-02,  1.5940e-01,  2.4880e-02, -6.4660e-02, -3.3957e-02,\n",
      "         2.4542e-02, -6.8554e-02,  6.3840e-02,  9.2587e-02, -1.9237e-02,\n",
      "        -6.5632e-02, -2.1189e-03, -2.8654e-02,  1.1331e-01,  1.6623e-02,\n",
      "         1.1080e-02,  1.7222e-02,  2.1639e-02,  1.2736e-02,  5.2551e-03,\n",
      "        -9.9089e-02, -2.6358e-02, -9.2213e-02,  1.2136e-01, -5.5554e-03,\n",
      "         1.1617e-02, -4.7833e-02,  7.7265e-02, -3.7647e-02,  5.4385e-02,\n",
      "         2.0990e-02,  2.2553e-02, -6.3382e-02, -5.5776e-02, -5.7225e-02,\n",
      "         9.2727e-02,  1.6663e-02,  6.5102e-02, -1.2185e-03,  5.3320e-02,\n",
      "        -4.8457e-02,  7.5062e-02, -1.4598e-01,  3.8588e-02, -3.4603e-02,\n",
      "         8.2583e-02, -1.1695e-01,  6.2174e-02, -3.0089e-02,  6.0242e-02,\n",
      "        -8.7868e-02,  2.0539e-02,  2.1731e-02,  5.3997e-02, -1.3505e-01,\n",
      "         1.0796e-01,  2.6054e-02, -2.6334e-03, -3.0498e-02,  5.7488e-02,\n",
      "         2.5467e-02,  1.6457e-01, -1.1166e-01,  6.3687e-02, -7.1224e-02,\n",
      "         4.8232e-02,  3.7671e-02, -3.7511e-02,  6.1815e-03,  5.9239e-02,\n",
      "        -4.3058e-02,  5.7244e-02, -3.7723e-02, -4.0530e-02, -1.6016e-04,\n",
      "        -7.7277e-02,  4.9145e-02,  1.0879e-01, -7.0079e-02,  4.3076e-02,\n",
      "        -7.8011e-02,  8.8930e-02,  2.0812e-02, -4.1997e-02, -8.8763e-02,\n",
      "         7.2150e-02, -1.2089e-01,  4.8207e-02, -1.6290e-01,  2.8618e-02,\n",
      "        -4.6027e-02, -4.2223e-02, -5.9502e-02, -2.3213e-02, -5.4078e-02,\n",
      "         2.0160e-02, -4.0111e-02,  6.2484e-02, -9.7627e-02,  1.1721e-01,\n",
      "        -1.6134e-02,  5.0450e-02, -4.9904e-02,  1.2698e-01, -5.4202e-02,\n",
      "         1.6661e-01,  2.1847e-02,  5.4467e-02, -4.3526e-02, -3.2171e-02,\n",
      "        -1.2111e-01,  4.7303e-02, -1.2938e-01, -1.9988e-02, -5.1697e-02,\n",
      "         6.1393e-02, -2.3137e-02,  5.6133e-02, -1.4154e-02, -6.1242e-02,\n",
      "         4.4444e-02, -1.0746e-02, -1.2849e-01, -8.0400e-03, -1.1127e-01,\n",
      "         1.1649e-01,  2.5405e-02,  3.6932e-02,  3.8607e-03,  6.1090e-03,\n",
      "         8.7280e-03, -6.1128e-02, -5.6000e-02,  8.2048e-02, -5.5223e-02,\n",
      "        -3.3294e-02,  1.4853e-02,  5.3807e-02, -4.7080e-02])), ('bert.encoder.layer.3.attention.self.query.weight', tensor([[-0.0285, -0.0342, -0.0309,  ..., -0.0099,  0.0102,  0.0033],\n",
      "        [-0.0194, -0.0428, -0.0057,  ...,  0.0034,  0.0048,  0.0609],\n",
      "        [ 0.0856, -0.0223,  0.0555,  ..., -0.0756, -0.0088, -0.1380],\n",
      "        ...,\n",
      "        [-0.0289,  0.0027,  0.0415,  ...,  0.0160, -0.0036,  0.0175],\n",
      "        [-0.1041,  0.0645,  0.0085,  ...,  0.0077, -0.0043,  0.0430],\n",
      "        [ 0.0008,  0.0449,  0.0027,  ..., -0.0677, -0.0737,  0.0625]])), ('bert.encoder.layer.3.attention.self.query.bias', tensor([-0.0226,  0.0195,  0.0022,  0.0119, -0.0068, -0.0130, -0.0407, -0.0357,\n",
      "         0.0095,  0.0365, -0.0222, -0.0137, -0.0162,  0.0041,  0.0081, -0.0130,\n",
      "        -0.0058,  0.0374, -0.0123, -0.1205, -0.0413,  0.0165,  0.0167,  0.0470,\n",
      "        -0.0053, -0.0099, -0.0379, -0.0233,  0.0011,  0.0284, -0.0255,  0.0086,\n",
      "        -0.0006, -0.0019,  0.0279,  0.0461,  0.0252, -0.0697,  0.1308,  0.1321,\n",
      "        -0.0646, -0.0277, -0.0050, -0.0565,  0.0603, -0.0789, -0.0287, -0.0956,\n",
      "         0.0356, -0.0474, -0.0752,  0.0221,  0.0880,  0.0736,  0.0068, -0.0333,\n",
      "        -0.0211,  0.0349, -0.0654,  0.0600, -0.0257, -0.0127,  0.0221,  0.1265,\n",
      "         0.0145,  0.0233,  0.0253,  0.0034,  0.0131,  0.0089, -0.0010, -0.0218,\n",
      "         0.0114, -0.0115,  0.0088, -0.0267, -0.0090,  0.0088,  0.0048, -0.0225,\n",
      "         0.0061, -0.0207, -0.0160,  0.0240,  0.0438, -0.0156,  0.0026, -0.0484,\n",
      "         0.0093, -0.0023,  0.0157,  0.0724,  0.0158,  0.0142, -0.0112,  0.0156,\n",
      "         0.0217, -0.0414,  0.0180, -0.0101,  0.0077,  0.0276,  0.0188,  0.0098,\n",
      "        -0.0082,  0.0010,  0.0311, -0.0414, -0.0390,  0.0377, -0.0070, -0.0138,\n",
      "        -0.0383, -0.0160,  0.0632, -0.0350, -0.0685,  0.0242,  0.0274, -0.0111,\n",
      "        -0.0520,  0.0797,  0.0228, -0.0262, -0.0400, -0.0031,  0.0294, -0.0228,\n",
      "        -0.0174, -0.1927,  0.0559,  0.0717,  0.0059,  0.0198, -0.0663, -0.0363,\n",
      "        -0.0049,  0.1191,  0.0893, -0.1150, -0.0727,  0.1901,  0.0007,  0.0371,\n",
      "        -0.0864,  0.1282,  0.0615, -0.0525, -0.1799, -0.0775, -0.1160, -0.1153,\n",
      "         0.0495, -0.0004,  0.0950, -0.1313, -0.1530,  0.1765,  0.0134, -0.0519,\n",
      "         0.0019,  0.0310,  0.0379, -0.0340, -0.0297, -0.0125,  0.0940, -0.0067,\n",
      "         0.0151, -0.0295,  0.0861,  0.0412,  0.0049,  0.0057,  0.0145, -0.0561,\n",
      "        -0.0397, -0.0349, -0.0282,  0.0402,  0.0519, -0.0707,  0.0401, -0.0845,\n",
      "         0.0111,  0.0227, -0.0139, -0.0553, -0.0173,  0.0693,  0.0580,  0.0069,\n",
      "        -0.0925, -0.0066, -0.0017, -0.0608, -0.0528, -0.0064, -0.0495,  0.0316,\n",
      "        -0.0164, -0.0551, -0.0526,  0.0134, -0.0442,  0.0004,  0.0181,  0.0005,\n",
      "        -0.0113, -0.0295, -0.0423, -0.0266, -0.0040,  0.0051, -0.0047,  0.0023,\n",
      "        -0.0330, -0.0336, -0.0192,  0.0034, -0.0042, -0.0445, -0.0173,  0.0038,\n",
      "        -0.0223, -0.0154,  0.0204,  0.0122, -0.0273,  0.0023, -0.0161,  0.0053,\n",
      "        -0.0455, -0.0662,  0.0136, -0.0294, -0.0074,  0.0392, -0.0775,  0.0122,\n",
      "        -0.0020,  0.0227, -0.0409,  0.0192,  0.0190, -0.0384, -0.0216,  0.0074,\n",
      "        -0.0109, -0.0378,  0.0316,  0.0122, -0.0469,  0.0065,  0.0105,  0.0078,\n",
      "        -0.0249,  0.0253, -0.0376,  0.0490, -0.0128, -0.0053,  0.0839, -0.0449,\n",
      "         0.0196, -0.0127,  0.0912,  0.0138, -0.0271, -0.0352,  0.0020, -0.0033,\n",
      "        -0.0502, -0.0485,  0.0563, -0.0300, -0.0051, -0.0236,  0.0507,  0.0118,\n",
      "         0.0278, -0.0120, -0.0267,  0.0158, -0.0449, -0.0196,  0.0181, -0.0279,\n",
      "        -0.0347,  0.0469, -0.0448,  0.0698,  0.0219,  0.0312,  0.0055, -0.0357,\n",
      "        -0.0060, -0.0062, -0.0015,  0.0620, -0.0126, -0.0218,  0.0685,  0.0551,\n",
      "         0.0173, -0.0321,  0.0896, -0.0017, -0.0092,  0.0477,  0.0669, -0.0131,\n",
      "         0.0490,  0.0107,  0.0578, -0.0074,  0.0242, -0.0397,  0.0618,  0.0695,\n",
      "         0.0189, -0.0220,  0.0022,  0.0018,  0.0093,  0.0064,  0.0010,  0.0468,\n",
      "        -0.0564,  0.0568,  0.0032,  0.0277,  0.0491,  0.0414,  0.0293, -0.0480,\n",
      "         0.0005, -0.0173,  0.0718,  0.0267,  0.0148,  0.0317, -0.0276, -0.0248,\n",
      "         0.0023,  0.0428, -0.0194,  0.0511, -0.0044, -0.0228,  0.0008, -0.0223,\n",
      "        -0.0437, -0.0342,  0.0116, -0.0195, -0.0121,  0.0479,  0.0130,  0.0209,\n",
      "         0.0074,  0.0573, -0.0080, -0.0065,  0.0520,  0.0147,  0.0190, -0.0023,\n",
      "        -0.0308, -0.0191, -0.0812, -0.0062,  0.0342, -0.0218,  0.0064, -0.0218,\n",
      "        -0.0316,  0.0052, -0.0397, -0.0679,  0.0701,  0.0035, -0.0134, -0.0138])), ('bert.encoder.layer.3.attention.self.key.weight', tensor([[ 0.0231,  0.0455, -0.0169,  ...,  0.0006,  0.0113,  0.0065],\n",
      "        [-0.0526, -0.0503,  0.0153,  ...,  0.0288,  0.0320,  0.0548],\n",
      "        [ 0.0556,  0.0469, -0.0169,  ..., -0.0211, -0.0491, -0.0013],\n",
      "        ...,\n",
      "        [-0.0167,  0.0769,  0.0305,  ..., -0.0078,  0.0358,  0.0128],\n",
      "        [ 0.0390, -0.0382,  0.0174,  ..., -0.0344, -0.0165,  0.0269],\n",
      "        [-0.0513,  0.0332,  0.0673,  ..., -0.0699,  0.0774, -0.0162]])), ('bert.encoder.layer.3.attention.self.key.bias', tensor([ 2.8804e-04,  1.1949e-03,  5.4245e-04,  2.6636e-04, -1.4157e-03,\n",
      "         7.1764e-04,  6.7434e-04,  2.1689e-04, -9.1120e-04,  1.7687e-03,\n",
      "        -2.1364e-04,  2.5887e-04, -7.2184e-04,  5.7599e-04,  2.3083e-04,\n",
      "        -1.2911e-04, -7.8970e-04,  3.2235e-04,  4.3036e-04,  2.7903e-03,\n",
      "         1.2651e-03, -6.9123e-04,  3.8893e-05,  1.0249e-03,  4.1671e-04,\n",
      "        -1.1774e-03,  1.1481e-03, -6.8169e-05,  6.7326e-04, -2.0465e-03,\n",
      "        -5.3162e-04, -7.6182e-04, -1.4595e-03,  4.9677e-04,  1.3243e-03,\n",
      "         1.3469e-04, -1.5910e-03, -4.3945e-05, -7.8334e-04, -8.9642e-05,\n",
      "        -1.0008e-03,  1.3019e-04, -1.2536e-04,  1.9003e-03, -8.0864e-04,\n",
      "         1.1709e-03,  4.8225e-04,  2.2147e-03,  2.2857e-03, -1.5724e-03,\n",
      "        -3.2828e-04, -2.3759e-03,  1.8422e-03,  3.4897e-04, -1.6540e-03,\n",
      "         7.5766e-04, -1.1052e-03, -2.1728e-04, -2.5354e-03, -6.3118e-04,\n",
      "        -2.6865e-04, -1.2356e-03,  1.5190e-03,  9.6905e-04,  1.1265e-03,\n",
      "        -1.2190e-03,  1.9607e-03,  5.5860e-04, -8.9933e-04, -5.1910e-04,\n",
      "        -2.1706e-04, -4.9149e-04,  3.9726e-03,  5.7371e-04,  2.6326e-04,\n",
      "         1.1473e-03,  3.5949e-04,  7.2699e-04,  1.5713e-03, -2.7685e-03,\n",
      "         2.7582e-04, -2.9197e-05, -7.4668e-05, -1.8103e-03,  1.1658e-03,\n",
      "        -1.0414e-03,  3.0831e-04, -1.3004e-03,  2.1353e-03,  8.6371e-05,\n",
      "        -3.2894e-03,  2.3857e-03,  4.6711e-04,  9.1161e-05,  8.1034e-04,\n",
      "         1.0704e-03, -9.6298e-04, -5.8507e-04, -2.0425e-03, -2.9792e-04,\n",
      "        -1.0339e-03, -5.0391e-05, -1.3775e-03,  4.2856e-04,  1.4335e-04,\n",
      "         4.4786e-04, -7.5427e-04, -9.9911e-04,  4.5453e-04, -1.5142e-03,\n",
      "         5.9368e-04, -1.1824e-03, -6.7007e-04,  1.1396e-04,  6.9046e-04,\n",
      "         2.7009e-04, -3.2832e-04,  1.3440e-03,  8.2838e-04, -7.4332e-05,\n",
      "         6.4208e-05,  1.7840e-04, -1.1756e-04,  4.4636e-04, -7.3611e-04,\n",
      "        -9.1055e-04,  2.0765e-03, -1.2893e-03,  4.8132e-03, -4.1219e-03,\n",
      "         3.0413e-03,  6.8393e-03, -4.8782e-03,  6.0170e-03,  4.6098e-03,\n",
      "         2.0579e-03,  5.3189e-03,  3.0117e-03, -9.0378e-03,  1.4271e-02,\n",
      "         1.6853e-02, -2.7780e-02,  4.2378e-03, -1.2498e-02, -1.3255e-03,\n",
      "         8.7716e-03,  5.4376e-03,  4.0628e-03,  1.8461e-02,  1.0625e-02,\n",
      "         6.9121e-03,  4.7411e-03, -7.8819e-03, -8.5748e-03, -3.4689e-03,\n",
      "         2.1996e-02,  4.4672e-03, -6.1682e-03, -8.2837e-04,  1.5086e-02,\n",
      "         6.4445e-04,  2.1422e-03, -1.4110e-03,  5.0120e-04,  3.3578e-05,\n",
      "         1.3816e-04,  1.3022e-03,  4.2027e-03,  1.4873e-04, -3.4037e-03,\n",
      "         7.4342e-04,  8.0063e-04,  6.3359e-06, -2.1543e-04, -1.4210e-04,\n",
      "        -3.4756e-04, -3.1856e-03,  2.4631e-03, -1.4498e-03, -5.3451e-05,\n",
      "        -7.3023e-04, -2.6715e-03, -6.2316e-04,  6.5716e-04,  1.1289e-04,\n",
      "        -1.8330e-03,  1.9691e-03, -1.3633e-03, -5.0042e-04, -1.5898e-03,\n",
      "         3.4989e-03,  2.2260e-04,  5.6484e-04, -7.2051e-05,  5.4717e-04,\n",
      "        -1.1387e-04, -5.1310e-05,  5.9426e-04, -1.2041e-04,  1.0664e-03,\n",
      "        -1.5005e-04, -1.5504e-03, -7.4192e-05,  9.0909e-04, -7.1289e-04,\n",
      "         4.6925e-04,  1.8426e-03, -1.4039e-03, -4.2984e-04,  1.8321e-04,\n",
      "        -9.7186e-04, -9.4790e-04,  6.5439e-04, -1.9707e-03, -1.1205e-03,\n",
      "        -1.7965e-03,  1.1310e-03,  3.3751e-04, -1.7950e-04, -9.7976e-04,\n",
      "         1.3273e-03,  2.6303e-04,  7.9894e-05, -1.4580e-04, -1.4090e-03,\n",
      "         1.7047e-03,  7.1445e-04,  1.1668e-03, -1.9751e-03, -4.5531e-04,\n",
      "         3.6588e-04,  5.0534e-06, -1.1514e-03,  4.0500e-03,  7.7518e-04,\n",
      "        -1.3842e-03,  1.2779e-04, -2.4992e-04,  7.5278e-04, -1.3557e-03,\n",
      "         4.8282e-04, -1.4882e-03,  3.4478e-04,  5.9962e-04,  1.9344e-03,\n",
      "         4.6169e-04, -6.6654e-04,  7.4236e-04,  1.4733e-03, -1.0616e-03,\n",
      "        -7.6612e-04, -1.3800e-04,  1.5341e-03,  4.3118e-05, -4.4796e-04,\n",
      "        -2.9714e-04, -4.3731e-05, -9.1598e-04, -1.3976e-03, -1.8342e-03,\n",
      "        -4.3450e-04,  7.8361e-04, -1.6375e-03, -4.0976e-03,  1.4089e-03,\n",
      "        -2.7264e-03,  1.1004e-03, -9.9282e-06,  2.2847e-03, -6.5031e-04,\n",
      "         1.9405e-04,  4.4231e-03, -1.1679e-03, -4.1806e-04, -4.3252e-03,\n",
      "         8.1197e-04,  4.0679e-03, -1.1678e-03, -1.0384e-04,  8.1441e-04,\n",
      "         3.1348e-03, -2.8394e-03,  1.2456e-03,  3.9250e-03,  1.6182e-03,\n",
      "        -1.6865e-03, -1.8892e-03, -2.1905e-03, -2.1536e-03, -8.6924e-04,\n",
      "        -1.9973e-04,  7.3855e-04,  4.5161e-04, -1.1114e-03, -4.3407e-03,\n",
      "         8.1008e-04,  2.1916e-04,  1.4896e-04, -4.5365e-05, -2.5575e-03,\n",
      "         1.1621e-03,  2.3033e-03,  9.9507e-04, -1.3793e-03,  1.3795e-03,\n",
      "        -1.7579e-03, -7.2635e-04, -5.3232e-04, -6.6249e-04, -3.7602e-04,\n",
      "        -2.3275e-03,  2.0541e-03,  1.1420e-03, -2.5196e-04, -1.7009e-04,\n",
      "         1.2044e-03,  4.9350e-05,  1.0251e-03,  8.3694e-04, -1.4885e-03,\n",
      "        -4.3869e-06,  1.2364e-03, -8.3661e-04,  9.5036e-04, -1.4013e-03,\n",
      "        -1.8665e-03,  6.7532e-04, -1.5944e-03,  7.7146e-05,  6.9943e-04,\n",
      "         1.5269e-03, -7.2612e-04, -3.2648e-04, -1.2095e-03,  5.7668e-04,\n",
      "         4.6200e-05, -8.8365e-04,  8.0293e-04, -1.4511e-03,  4.8296e-04,\n",
      "        -1.8973e-03, -2.2007e-03,  6.0856e-04,  8.9224e-04, -8.3134e-04,\n",
      "         1.0591e-03,  1.5199e-03,  4.3667e-05, -1.5252e-03, -1.4551e-03,\n",
      "        -6.7277e-04,  1.1614e-03, -2.2886e-03,  3.1582e-03, -5.6113e-04,\n",
      "        -2.1249e-03,  7.3048e-04,  1.5391e-03, -3.3458e-04,  1.9384e-03,\n",
      "        -1.7367e-03, -2.6335e-03, -8.9895e-04,  3.4436e-04,  1.5002e-03,\n",
      "         4.8953e-04, -4.3511e-03,  3.0822e-03, -8.6644e-04,  2.3544e-04,\n",
      "         1.7036e-03,  9.2095e-04,  9.8176e-04,  1.7030e-03,  3.9075e-03,\n",
      "        -8.3477e-05, -2.1703e-03, -7.2491e-04,  6.3305e-04, -3.3536e-03,\n",
      "         7.9765e-04, -1.9460e-03, -1.5707e-03, -2.0055e-03])), ('bert.encoder.layer.3.attention.self.value.weight', tensor([[ 0.0383,  0.0181,  0.0502,  ...,  0.0198,  0.0238,  0.0300],\n",
      "        [ 0.0058,  0.0475,  0.0752,  ...,  0.0184, -0.0408, -0.0028],\n",
      "        [-0.0196,  0.0143,  0.0206,  ..., -0.0027,  0.0288,  0.0710],\n",
      "        ...,\n",
      "        [-0.0554, -0.0411,  0.0491,  ...,  0.0169, -0.0081, -0.0142],\n",
      "        [-0.0005, -0.0776,  0.0112,  ..., -0.1076,  0.0280, -0.0063],\n",
      "        [-0.0937, -0.0298, -0.0355,  ...,  0.0081, -0.0087,  0.0232]])), ('bert.encoder.layer.3.attention.self.value.bias', tensor([-1.4017e-02, -4.7434e-02,  2.5503e-03,  1.4361e-02, -2.4146e-02,\n",
      "         1.8205e-02,  4.0504e-02, -3.3725e-02, -2.8091e-02,  9.1260e-03,\n",
      "        -3.6183e-02,  2.9007e-02,  1.7978e-03, -1.1766e-02, -3.1354e-02,\n",
      "         2.2815e-02, -3.9401e-02, -1.8797e-02, -1.3105e-02,  2.0253e-02,\n",
      "        -1.8157e-02, -3.5928e-02,  1.2710e-02, -1.3656e-02, -1.9674e-02,\n",
      "         1.6362e-02, -2.8323e-02,  1.5592e-02, -2.4276e-02, -2.4824e-02,\n",
      "        -5.5166e-03, -2.9456e-02,  2.9463e-02, -1.7676e-02,  1.9191e-02,\n",
      "         2.6890e-03,  2.7666e-03,  1.4626e-02,  4.5713e-03, -1.1691e-02,\n",
      "         1.7408e-03, -8.3158e-03, -1.3062e-02,  3.9830e-04, -6.3561e-03,\n",
      "         4.3141e-03, -8.0536e-03, -3.3139e-03, -6.5632e-03,  2.6205e-03,\n",
      "         1.3399e-02,  1.2033e-02,  2.0472e-02,  1.3088e-03, -2.4349e-02,\n",
      "         1.3883e-02,  2.5152e-02, -2.2479e-02,  1.7410e-02,  1.4424e-02,\n",
      "        -1.3616e-02, -1.5529e-02,  1.5731e-02, -1.6962e-03, -1.6152e-02,\n",
      "         2.2926e-02, -2.5481e-02,  2.6202e-02,  8.3760e-03, -8.1707e-03,\n",
      "         2.0928e-02, -1.5625e-02,  1.0195e-02, -8.4838e-03, -1.7859e-02,\n",
      "         2.5608e-02,  2.8389e-02,  1.5445e-02,  7.1575e-03,  1.7323e-02,\n",
      "        -1.5387e-02,  2.2193e-03,  2.0434e-03,  3.7561e-03,  4.1008e-02,\n",
      "         2.6289e-03,  1.2937e-02,  1.8443e-03, -5.8130e-03, -1.1256e-02,\n",
      "         1.8307e-02, -1.1457e-02, -1.5629e-02,  1.9120e-02,  1.3280e-02,\n",
      "        -8.8912e-03, -2.8400e-02,  4.7334e-03, -1.0156e-02, -1.0089e-02,\n",
      "        -2.8249e-02,  2.0428e-03, -8.3104e-03, -3.9428e-03,  1.0804e-02,\n",
      "         3.6056e-02, -8.0183e-03,  5.1136e-03,  1.6252e-02, -4.7460e-03,\n",
      "        -8.1724e-03,  4.9120e-03,  6.3886e-03, -2.7992e-03, -5.0634e-03,\n",
      "         2.6045e-03, -1.5399e-02, -2.0423e-02,  3.9823e-02, -3.8116e-03,\n",
      "         7.4424e-04,  6.2436e-03, -2.2502e-03,  4.1523e-03,  1.0838e-02,\n",
      "         1.6853e-02, -1.3379e-02,  5.3880e-03,  4.0755e-02,  5.7802e-02,\n",
      "         1.2692e-01, -1.0243e-02,  1.0586e-02,  8.2493e-02, -4.5168e-02,\n",
      "        -1.2243e-01, -8.8449e-02, -4.1410e-02, -4.1365e-02,  2.6446e-02,\n",
      "        -1.0991e-01,  1.5183e-01, -1.8989e-02,  3.3284e-02, -3.2162e-02,\n",
      "         1.1466e-01, -5.4716e-02,  5.6323e-02,  3.8983e-02,  2.3900e-02,\n",
      "         1.3668e-02, -4.3761e-02, -3.9719e-02, -1.3649e-02, -2.4317e-02,\n",
      "        -1.1192e-01,  4.9151e-03, -4.3919e-02,  2.5461e-02,  3.4111e-02,\n",
      "        -9.9330e-04,  1.6181e-02,  2.9957e-02, -1.5401e-02,  2.7631e-02,\n",
      "         6.6066e-02, -1.2923e-02, -1.6659e-02,  1.1632e-02, -4.0222e-02,\n",
      "        -4.5699e-02,  2.0079e-02,  2.3930e-02, -3.6809e-02, -2.8771e-03,\n",
      "        -2.3631e-02,  3.8111e-02,  3.4669e-02,  1.2006e-02, -1.9285e-02,\n",
      "         6.9069e-03,  7.1390e-02, -4.7776e-02,  1.6855e-03,  3.4169e-03,\n",
      "        -7.8979e-03, -1.1007e-02, -2.1493e-02,  3.7040e-02, -3.2892e-03,\n",
      "        -2.8327e-02, -3.1187e-02,  1.3770e-02, -9.7497e-03, -1.1993e-02,\n",
      "         1.4495e-03, -6.1109e-03,  6.7007e-03,  2.5824e-03,  2.3852e-03,\n",
      "        -1.4464e-02,  2.6910e-02, -7.1959e-03, -1.0611e-02, -1.7312e-02,\n",
      "        -8.4376e-04,  2.9794e-02,  3.9168e-03,  9.1542e-03, -1.4289e-02,\n",
      "        -1.0980e-02,  2.8814e-03,  8.0996e-04, -1.7523e-03, -2.1671e-02,\n",
      "        -1.3425e-02,  5.8859e-05, -4.9328e-03, -4.2765e-04,  8.9860e-04,\n",
      "         1.0284e-03,  1.5702e-03,  1.6942e-02,  7.5105e-03,  8.8738e-03,\n",
      "         4.5192e-04, -6.1913e-03,  2.7593e-03, -5.6517e-03,  1.7860e-02,\n",
      "         4.8163e-03,  1.6766e-02, -1.7578e-02,  1.1557e-02,  5.3621e-02,\n",
      "         3.2193e-02, -3.0757e-02, -1.4320e-02, -1.6209e-03,  3.0761e-02,\n",
      "         1.1187e-02,  4.1118e-04,  3.8306e-02, -2.7167e-02, -2.4904e-02,\n",
      "        -2.0296e-02,  2.5352e-02, -1.4523e-02,  2.1866e-03, -2.4668e-02,\n",
      "        -4.0252e-02, -1.7320e-02,  1.1373e-02,  1.6860e-02,  1.0741e-02,\n",
      "         2.8119e-03,  1.4792e-02, -1.5666e-03, -2.5150e-02, -2.1003e-02,\n",
      "        -3.1408e-02,  2.6067e-02, -9.6576e-03, -6.4913e-03, -1.3268e-02,\n",
      "         1.0036e-02, -1.7124e-02, -3.4703e-02,  3.3564e-02,  1.0943e-02,\n",
      "         9.3975e-03, -2.5105e-02,  1.4605e-02,  1.4836e-02, -1.2169e-02,\n",
      "        -1.7767e-02,  3.1303e-03,  9.1973e-03,  3.1402e-02,  1.5798e-02,\n",
      "         2.0943e-02, -9.5292e-03, -1.8252e-02, -1.4275e-03,  3.4396e-02,\n",
      "        -1.4618e-03, -5.0072e-03,  3.1928e-02, -1.4403e-02,  7.4396e-03,\n",
      "        -6.3980e-03, -1.1457e-02,  9.9856e-03,  1.1675e-02,  1.6370e-03,\n",
      "         3.7771e-03, -1.7683e-02,  1.3514e-02,  2.5387e-03, -2.1503e-02,\n",
      "        -8.9943e-03,  7.7891e-03, -1.2618e-02,  2.0359e-02,  4.9440e-03,\n",
      "         3.1429e-02,  1.1637e-02,  5.5784e-03,  1.2755e-02,  1.3241e-02,\n",
      "        -3.5367e-03, -5.3406e-03,  1.5284e-03, -7.7609e-04, -1.4951e-03,\n",
      "         1.2591e-02,  2.3631e-02,  9.6503e-04,  9.1599e-03,  8.8474e-03,\n",
      "        -5.3119e-03, -8.8468e-04,  7.4218e-03,  1.0055e-02,  2.5955e-03,\n",
      "        -5.8204e-03,  8.4226e-03, -1.2704e-02,  4.6940e-03,  1.3358e-02,\n",
      "        -9.1904e-03,  7.1485e-04,  2.6816e-03, -7.4995e-03,  8.4977e-03,\n",
      "         2.7769e-03,  1.4916e-02, -1.4538e-02,  2.7514e-03,  1.0995e-02,\n",
      "        -6.0958e-03, -9.4335e-03, -1.1269e-02,  1.0265e-02, -1.0664e-02,\n",
      "        -1.1077e-02,  1.0113e-02, -1.0887e-02, -2.2265e-03, -4.7198e-04,\n",
      "        -1.4435e-03, -4.8900e-03, -2.2802e-02, -8.5232e-03,  9.6531e-03,\n",
      "         6.6860e-03,  2.0332e-02, -1.3266e-02, -4.4306e-03,  1.8069e-02,\n",
      "        -7.0678e-03, -1.1046e-02, -1.7007e-02, -4.9820e-03, -9.9242e-03,\n",
      "        -3.6763e-03, -5.2186e-03, -6.4454e-03,  4.7403e-03,  8.6990e-03,\n",
      "        -1.9253e-03,  4.4503e-03,  9.6230e-03, -1.1979e-02,  7.5273e-03,\n",
      "        -8.9688e-03,  1.0023e-02, -1.5556e-02,  3.4661e-03,  1.0783e-03,\n",
      "         3.6128e-03, -1.1058e-02, -8.0897e-03,  1.2471e-02])), ('bert.encoder.layer.3.attention.output.dense.weight', tensor([[-0.0061, -0.1043,  0.0108,  ..., -0.0620, -0.0747, -0.0050],\n",
      "        [ 0.0486,  0.0538, -0.0089,  ...,  0.0142,  0.0421,  0.0569],\n",
      "        [-0.0370, -0.0649,  0.0442,  ...,  0.0341, -0.0481,  0.0710],\n",
      "        ...,\n",
      "        [-0.0151, -0.0383, -0.0413,  ...,  0.0047, -0.0138, -0.0180],\n",
      "        [-0.0327, -0.0373,  0.0145,  ..., -0.0390, -0.0394, -0.0228],\n",
      "        [-0.0012,  0.0703,  0.0122,  ...,  0.0265,  0.0201,  0.0136]])), ('bert.encoder.layer.3.attention.output.dense.bias', tensor([-0.0430, -0.0242, -0.0103, -0.0607,  0.0855, -0.0724,  0.1083, -0.0686,\n",
      "        -0.1054, -0.0296,  0.0587, -0.0432, -0.0566, -0.0692, -0.0383, -0.0049,\n",
      "        -0.0692,  0.0013, -0.0253,  0.0418, -0.0003,  0.0301, -0.0310,  0.1202,\n",
      "         0.0314, -0.0436,  0.0568,  0.0253, -0.0359, -0.0643, -0.0210, -0.0105,\n",
      "        -0.0511, -0.0098, -0.1183,  0.0420,  0.0540, -0.0968, -0.0106, -0.0308,\n",
      "         0.0575, -0.0670, -0.0216, -0.0905,  0.0672,  0.0645, -0.0145,  0.0817,\n",
      "        -0.0147, -0.0404, -0.0187, -0.0350,  0.0179,  0.0745, -0.0339,  0.0315,\n",
      "        -0.0523,  0.0908, -0.1313, -0.1042, -0.0286,  0.0268, -0.0022,  0.0670,\n",
      "         0.0313, -0.0112,  0.0225,  0.0381,  0.0077,  0.0384,  0.0014,  0.0115,\n",
      "         0.0068,  0.0484, -0.0559, -0.0165,  0.0712,  0.0028, -0.0002, -0.0064,\n",
      "         0.0481,  0.0972,  0.0745, -0.0658, -0.0560, -0.0518, -0.0300,  0.0448,\n",
      "        -0.0917,  0.0584, -0.0182, -0.0180, -0.0287,  0.0635,  0.0258,  0.0743,\n",
      "         0.0070,  0.0440, -0.0486,  0.0273, -0.0303,  0.0285,  0.0102,  0.0555,\n",
      "        -0.0074, -0.0435, -0.0111,  0.0130,  0.0493,  0.0582,  0.0420, -0.0098,\n",
      "        -0.0482,  0.0458,  0.0216,  0.0590,  0.0491, -0.0283,  0.0615, -0.0691,\n",
      "         0.0645,  0.0266,  0.0063, -0.0232,  0.0546, -0.0971,  0.0194, -0.0476,\n",
      "        -0.0414,  0.0358,  0.0379,  0.0786,  0.0310, -0.0151,  0.0124,  0.0101,\n",
      "         0.0504, -0.0536, -0.0452, -0.0436, -0.0080,  0.0416, -0.0199,  0.0125,\n",
      "        -0.0440,  0.0021,  0.0219,  0.0251, -0.0234,  0.0401, -0.0069,  0.0353,\n",
      "         0.0093,  0.0064, -0.0251, -0.0078,  0.0099,  0.0095, -0.0335,  0.0812,\n",
      "        -0.0603,  0.0131,  0.0525,  0.0271, -0.0095,  0.0100, -0.0381,  0.0550,\n",
      "        -0.0174,  0.0234, -0.0188, -0.0645, -0.0489,  0.0281,  0.0059,  0.0262,\n",
      "        -0.0031,  0.0073, -0.0226,  0.0622, -0.0276,  0.0171,  0.0121,  0.0393,\n",
      "        -0.0528, -0.0276, -0.0055,  0.1459,  0.0263,  0.0104, -0.0323,  0.0265,\n",
      "        -0.0067, -0.0032, -0.0604,  0.0095,  0.0669, -0.0057, -0.0364,  0.0393,\n",
      "        -0.0823,  0.0118,  0.0268,  0.0009, -0.0389,  0.0435, -0.0416,  0.0692,\n",
      "        -0.0032,  0.0160,  0.0044, -0.0236, -0.0513,  0.0139, -0.0429,  0.0373,\n",
      "        -0.0032, -0.0519, -0.0715,  0.0095, -0.0456, -0.0332, -0.2228, -0.0340,\n",
      "        -0.0532,  0.0153,  0.0267,  0.0759,  0.0368,  0.0370, -0.0272, -0.0121,\n",
      "        -0.0137, -0.0176, -0.0105, -0.0193,  0.0358,  0.0039, -0.0432, -0.0068,\n",
      "         0.0153, -0.0077,  0.0475,  0.0132,  0.0067,  0.1014, -0.0468,  0.0401,\n",
      "        -0.0268, -0.0233, -0.0194,  0.0912, -0.0294, -0.1040,  0.0032,  0.0691,\n",
      "         0.0284,  0.0014, -0.0891,  0.0089,  0.0126, -0.0275,  0.0161,  0.0112,\n",
      "         0.0501,  0.0444, -0.0066,  0.0009, -0.0245, -0.0372, -0.0390,  0.0863,\n",
      "        -0.0428,  0.0134,  0.0287,  0.1645, -0.0528,  0.0428, -0.0460,  0.1062,\n",
      "        -0.0441, -0.0489, -0.0337, -0.0427, -0.0824,  0.0584, -0.0427,  0.0790,\n",
      "        -0.0153, -0.0160, -0.0181,  0.0067, -0.0292, -0.0355, -0.0049,  0.0513,\n",
      "        -0.0253,  0.0494, -0.0495,  0.0907, -0.0559, -0.0464,  0.0367,  0.0279,\n",
      "        -0.0283,  0.0247, -0.0676, -0.0014,  0.0231,  0.0107, -0.0157, -0.0326,\n",
      "         0.0525, -0.0109, -0.0824,  0.0651, -0.0534,  0.0257,  0.0132, -0.0085,\n",
      "        -0.0011, -0.0228,  0.0503,  0.0550, -0.0032,  0.0538, -0.0673, -0.0119,\n",
      "         0.0028,  0.0628, -0.0369,  0.0475, -0.0162,  0.0694, -0.0138,  0.0596,\n",
      "        -0.0841, -0.0249, -0.0110,  0.0058, -0.0155,  0.0316,  0.0159,  0.0429,\n",
      "        -0.0400,  0.0153, -0.0136, -0.0381, -0.0837,  0.0792, -0.0625,  0.0552,\n",
      "        -0.0580, -0.0016, -0.0366,  0.0161, -0.0745,  0.0496, -0.0433,  0.0860,\n",
      "        -0.0525,  0.0192,  0.0479,  0.0416, -0.0058, -0.0497,  0.0483, -0.0110,\n",
      "        -0.0570,  0.0317, -0.0388,  0.0223, -0.0429, -0.0243,  0.0434, -0.0058,\n",
      "         0.0526, -0.0069, -0.0627,  0.0659, -0.0123, -0.0740, -0.0303,  0.0399])), ('bert.encoder.layer.3.attention.output.LayerNorm.weight', tensor([0.8439, 0.8186, 0.7972, 0.7551, 0.8028, 0.8547, 0.8130, 0.7395, 0.8388,\n",
      "        0.7754, 0.8134, 0.7633, 0.7385, 0.8044, 0.8312, 0.8118, 0.8384, 0.7952,\n",
      "        0.7411, 0.7745, 0.7476, 0.7472, 0.8180, 0.7669, 0.7239, 0.7407, 0.7271,\n",
      "        0.7610, 0.8014, 0.7968, 0.8488, 0.8150, 0.8056, 0.7973, 0.8275, 0.8502,\n",
      "        0.7934, 0.8085, 0.7681, 0.8131, 0.8157, 0.8268, 0.8776, 0.8372, 0.8252,\n",
      "        0.8568, 0.8280, 0.8998, 0.8901, 0.8564, 0.8272, 0.7898, 0.7563, 0.8737,\n",
      "        0.8365, 0.8414, 0.8452, 0.8232, 0.8482, 0.9407, 0.9030, 0.8908, 0.9085,\n",
      "        0.9309, 0.9317, 0.9872, 0.9926, 0.9858, 0.9695, 0.9467, 0.9161, 0.9770,\n",
      "        0.9041, 0.9978, 1.0155, 0.9997, 0.9863, 0.9738, 1.0044, 0.9950, 1.0497,\n",
      "        0.9117, 0.9591, 0.9879, 0.9749, 0.9946, 1.0017, 1.0453, 0.9695, 1.0202,\n",
      "        0.9659, 1.0247, 1.0298, 0.9910, 0.9823, 0.9783, 0.9877, 0.9862, 0.9761,\n",
      "        1.0093, 0.9969, 0.8915, 0.9973, 0.9937, 1.0195, 0.8748, 0.9682, 0.9896,\n",
      "        0.9639, 1.0045, 0.9709, 0.9998, 0.9883, 1.0001, 1.0131, 0.9663, 0.9598,\n",
      "        1.0490, 0.9833, 0.9858, 1.4747, 1.0035, 0.9993, 1.0040, 0.9911, 0.9811,\n",
      "        1.0117, 0.9610, 0.9501, 1.0168, 0.9636, 1.0089, 0.9622, 1.0002, 0.9739,\n",
      "        1.0132, 0.9713, 0.9784, 0.9648, 0.9853, 0.9867, 1.0010, 1.0066, 1.0029,\n",
      "        0.9845, 0.9893, 0.9790, 1.0006, 0.9923, 1.0116, 1.0080, 0.9807, 0.9542,\n",
      "        1.0095, 0.9328, 1.0078, 0.9500, 0.9666, 0.9741, 0.9260, 0.9715, 0.9731,\n",
      "        0.9653, 1.0044, 0.9746, 0.9334, 0.9765, 0.9539, 0.9676, 0.9993, 0.9770,\n",
      "        1.0103, 0.9746, 1.0054, 0.9518, 0.9934, 0.9753, 0.9814, 0.9999, 1.0241,\n",
      "        1.0218, 0.9460, 0.9750, 0.9962, 0.9929, 1.0118, 0.9742, 1.0999, 1.0142,\n",
      "        0.9704, 1.0174, 0.9742, 1.0228, 0.9805, 1.0642, 0.9730, 0.9710, 0.9881,\n",
      "        0.9953, 0.9604, 1.0006, 1.0131, 0.9717, 0.9801, 0.9917, 0.9818, 0.9985,\n",
      "        0.9888, 1.1147, 0.9891, 1.0050, 0.9574, 0.9857, 1.0045, 1.0087, 1.0076,\n",
      "        0.9630, 1.0045, 0.9735, 1.0016, 1.0124, 1.0104, 1.1638, 0.9441, 1.0611,\n",
      "        1.0177, 0.9905, 1.0003, 0.9909, 0.9580, 1.0272, 0.9706, 1.0211, 0.9784,\n",
      "        0.9682, 1.0052, 0.9438, 0.9939, 0.9694, 0.9781, 0.9743, 1.0014, 1.0048,\n",
      "        0.9905, 0.9617, 1.0303, 0.9956, 0.9857, 1.0642, 0.9932, 1.0168, 1.0303,\n",
      "        1.0219, 1.0702, 0.9910, 1.0242, 1.0003, 1.0053, 1.0187, 1.0316, 0.9733,\n",
      "        0.9890, 0.9293, 1.0262, 0.9649, 1.0311, 0.9621, 0.9655, 0.9478, 1.0049,\n",
      "        0.9931, 1.0751, 1.0049, 0.9677, 0.9763, 1.2191, 0.9890, 0.9959, 1.0008,\n",
      "        1.0257, 1.0265, 0.9509, 0.9974, 0.9408, 0.9659, 1.0105, 0.9981, 0.9713,\n",
      "        0.9979, 0.9696, 1.0258, 0.9977, 1.0035, 0.9961, 0.9880, 0.9930, 0.9814,\n",
      "        0.9767, 1.0257, 1.0663, 1.0701, 0.9778, 0.9801, 0.9921, 0.9900, 0.9571,\n",
      "        1.0249, 0.9864, 0.9981, 0.9632, 1.0067, 1.0067, 0.9933, 0.9945, 1.0277,\n",
      "        1.0020, 1.0316, 0.9894, 0.9823, 1.0222, 1.0238, 1.0169, 1.0078, 1.0188,\n",
      "        0.9948, 1.0167, 1.0598, 1.0126, 0.9621, 0.9737, 0.9914, 0.9589, 0.9978,\n",
      "        0.9831, 0.9806, 1.1212, 1.0093, 0.9687, 0.9907, 0.9611, 1.0340, 0.9860,\n",
      "        0.9671, 0.9692, 1.0368, 1.0224, 0.9732, 1.0234, 1.2135, 1.0523, 1.0713,\n",
      "        1.0050, 0.9809, 0.9801, 0.9686, 1.0198, 1.0371, 1.0443, 1.0263, 1.0375,\n",
      "        1.0003, 0.9737, 0.9984, 0.9081, 1.0150, 1.0348, 0.9960, 0.9903, 1.0302,\n",
      "        0.8950, 1.0123, 1.0221, 0.9811, 0.9973, 0.9825, 0.9682, 0.9664, 0.9533,\n",
      "        0.9834, 0.9725, 0.9837, 1.0232, 1.0007, 0.9912])), ('bert.encoder.layer.3.attention.output.LayerNorm.bias', tensor([-5.2578e-02, -1.3749e-01,  4.1918e-02, -5.7069e-02,  7.3538e-02,\n",
      "        -8.5026e-02,  1.3001e-01, -8.4628e-02, -2.4361e-01, -8.7978e-02,\n",
      "         1.2907e-01, -1.1330e-01, -1.2492e-01, -1.5377e-01, -1.0159e-01,\n",
      "         3.9949e-02, -3.0578e-02,  2.7914e-02,  5.2195e-03,  8.5292e-03,\n",
      "        -3.5021e-02,  1.1363e-01, -2.2847e-02,  2.1161e-01,  7.8483e-02,\n",
      "        -2.1849e-02,  5.5214e-02,  2.0350e-02, -7.1271e-02, -6.6527e-02,\n",
      "        -7.8342e-02, -5.5072e-02, -1.1672e-01, -1.0503e-01, -2.7135e-01,\n",
      "         1.0698e-01,  1.2914e-01, -4.4634e-02, -9.6074e-02, -3.4693e-02,\n",
      "        -1.0007e-02, -7.1041e-02, -1.2566e-01, -1.7776e-01, -5.7113e-03,\n",
      "         4.5212e-02,  1.6356e-02,  1.6568e-01, -1.4581e-01,  1.0765e-01,\n",
      "         1.1964e-01,  5.7028e-03,  8.1037e-02,  1.6364e-01, -1.3687e-01,\n",
      "         4.9855e-02, -1.5106e-01,  6.8018e-02, -2.0804e-01, -1.6538e-01,\n",
      "        -1.0957e-01,  6.3184e-02, -8.4383e-02,  4.4656e-02,  9.4985e-02,\n",
      "         5.9541e-02,  8.8842e-02,  2.8280e-02,  8.7250e-02,  6.3705e-02,\n",
      "         7.1617e-02,  1.0927e-01,  2.9882e-02,  1.5100e-01, -1.2901e-01,\n",
      "         5.9753e-02,  1.1907e-01, -2.7924e-02, -5.3600e-02, -8.5443e-02,\n",
      "         1.5149e-01,  2.0146e-01,  1.7968e-01, -8.2971e-02, -7.6213e-02,\n",
      "        -1.4759e-01, -8.1303e-03,  2.0076e-01, -1.4158e-01,  2.2203e-01,\n",
      "         4.1128e-02, -1.4583e-01, -1.1677e-01,  1.0638e-01,  1.3084e-01,\n",
      "         8.0372e-02, -2.0528e-02,  6.6254e-02, -1.1881e-01,  7.6120e-02,\n",
      "        -4.1639e-02,  6.8895e-02,  1.4022e-02,  1.0098e-01, -6.6571e-02,\n",
      "        -2.5536e-02, -4.9626e-02,  1.5186e-01,  5.2713e-03,  1.8482e-01,\n",
      "         1.3339e-01,  1.1072e-01, -2.1220e-01,  1.5283e-01,  3.2619e-02,\n",
      "         8.0334e-02,  3.8367e-02, -1.0246e-01,  1.0715e-01, -1.0000e-01,\n",
      "         2.6647e-01,  1.0393e-01,  6.4129e-02,  1.7034e-02,  1.3228e-01,\n",
      "        -1.0221e-01,  5.8920e-03, -1.6009e-01, -2.8835e-02,  1.6348e-01,\n",
      "         6.7447e-02,  1.8946e-01, -1.6996e-02, -6.8548e-02,  7.4949e-02,\n",
      "         4.4836e-02,  5.4607e-02, -8.5611e-02, -9.5665e-02, -8.9042e-02,\n",
      "        -9.2512e-02,  1.2689e-01, -7.8566e-02, -7.2522e-02, -8.5179e-02,\n",
      "        -1.7005e-03, -2.1781e-02,  4.6872e-02, -4.5220e-02,  8.6798e-02,\n",
      "         7.6794e-02,  1.3745e-01,  2.6153e-03, -2.3898e-02, -1.4183e-02,\n",
      "        -5.2905e-02, -1.0555e-02,  8.6477e-02, -7.9869e-02,  1.8178e-01,\n",
      "        -8.1717e-02, -4.8351e-03,  5.0791e-02,  1.2850e-01, -2.7992e-02,\n",
      "         1.4961e-01, -4.7933e-02,  8.5366e-02, -6.5177e-03,  1.1983e-01,\n",
      "        -1.3265e-02, -1.6912e-01, -6.8057e-02,  7.5744e-02, -9.8529e-03,\n",
      "         4.8782e-02, -3.3101e-02,  1.1392e-01, -6.4038e-02,  2.2728e-01,\n",
      "        -7.8408e-02,  1.6435e-01, -3.5247e-02,  1.0954e-01, -2.0275e-01,\n",
      "        -5.1755e-02, -5.8518e-02,  1.7057e-01,  5.7311e-02,  1.1355e-01,\n",
      "        -1.0930e-01,  7.6070e-02, -8.9828e-02, -5.6515e-02, -2.6485e-01,\n",
      "         1.0845e-01,  1.6092e-01,  9.5009e-02, -1.0525e-01, -2.5631e-02,\n",
      "        -1.6164e-01, -1.6231e-02,  7.0138e-02,  5.9450e-02, -9.9943e-02,\n",
      "         1.2711e-01,  1.6415e-02,  1.5935e-01, -1.0240e-01,  1.0103e-01,\n",
      "        -8.0751e-02,  6.1634e-02, -8.5926e-02,  1.0572e-01, -6.6289e-02,\n",
      "         1.3591e-01, -5.2119e-02, -7.6805e-02, -1.4571e-01,  2.8711e-02,\n",
      "        -1.2473e-01, -9.8657e-02, -5.3259e-01, -6.1857e-02, -2.4370e-01,\n",
      "        -2.4242e-02,  1.4698e-01,  1.5373e-01,  8.7290e-02,  8.6346e-02,\n",
      "         8.9040e-03,  9.1988e-03, -1.2474e-01, -1.9334e-02,  2.0238e-02,\n",
      "        -1.0241e-03, -2.6160e-02, -1.6545e-03, -6.1862e-02, -1.5556e-02,\n",
      "        -2.8676e-02,  1.4760e-01, -1.4222e-02,  9.4462e-02,  2.4049e-02,\n",
      "         1.7101e-01, -1.2040e-01,  3.9728e-02, -1.5085e-01,  4.1664e-03,\n",
      "        -1.5150e-02,  1.9092e-01, -6.0865e-02, -1.7782e-01, -4.6382e-03,\n",
      "         1.3523e-01,  7.9909e-02, -1.9816e-02, -3.2871e-01, -6.3165e-02,\n",
      "         2.3838e-02, -7.1145e-02,  2.7523e-02,  6.8886e-02,  7.0135e-02,\n",
      "         1.5081e-01,  2.4010e-02,  2.8541e-02,  4.8602e-03, -2.3814e-02,\n",
      "        -1.0575e-01,  2.9446e-01, -1.3731e-01,  7.8383e-02,  3.6088e-02,\n",
      "         4.9948e-01, -1.3170e-01,  1.5111e-01, -1.2846e-01,  2.3550e-01,\n",
      "        -1.2218e-01, -3.5088e-02, -5.7560e-03, -6.5310e-02, -8.7554e-02,\n",
      "         1.4571e-01, -1.2937e-01,  2.4632e-01, -9.0300e-02, -8.4195e-02,\n",
      "        -1.1590e-01,  1.0968e-01, -1.4006e-01, -1.2819e-01, -2.4649e-02,\n",
      "         8.7717e-02, -2.7514e-02,  1.2650e-01, -2.2575e-01,  2.4522e-01,\n",
      "        -1.3313e-01, -1.7790e-01,  6.0339e-02, -2.4959e-02, -8.6459e-02,\n",
      "         1.1126e-01, -1.4231e-01,  9.7908e-02,  1.9847e-02,  7.7861e-02,\n",
      "        -1.2880e-01,  3.6334e-02,  7.3086e-02, -5.9453e-02, -1.3143e-01,\n",
      "         1.1056e-01, -1.8376e-01,  1.5036e-01,  7.2298e-02, -3.5091e-02,\n",
      "         4.7396e-02, -1.1085e-02, -2.3140e-02,  1.4731e-01, -3.2268e-02,\n",
      "         6.2506e-02, -1.2315e-01, -1.7016e-01,  6.1433e-02,  1.2127e-01,\n",
      "        -1.2573e-01,  7.7147e-02, -1.5721e-01,  1.4972e-01, -5.0594e-02,\n",
      "         2.4188e-01, -1.7401e-01,  6.1119e-02,  4.4608e-02,  9.1746e-02,\n",
      "         6.9949e-03,  1.0019e-01,  2.6191e-03,  9.8360e-02, -1.9326e-01,\n",
      "        -8.8281e-03, -4.1542e-02, -1.7493e-01, -4.1234e-01,  2.4668e-01,\n",
      "        -2.0720e-01,  1.5174e-01, -1.4925e-01, -3.6131e-02, -7.5277e-02,\n",
      "         1.5286e-01, -1.5999e-01,  1.1262e-01, -1.2751e-01,  2.6850e-01,\n",
      "        -1.8270e-01,  2.6422e-02,  4.6895e-02,  7.3231e-02, -4.2786e-02,\n",
      "        -2.1429e-01,  9.2606e-02,  2.0662e-03, -1.9906e-01,  4.1985e-02,\n",
      "        -1.2179e-01, -3.6779e-02, -4.7621e-02, -1.9697e-03,  1.1274e-01,\n",
      "        -1.9508e-04,  8.4215e-02, -1.0072e-01, -1.3876e-01,  1.8425e-01,\n",
      "         2.2783e-02, -1.8288e-01, -1.5234e-01,  1.1924e-01])), ('bert.encoder.layer.3.intermediate.dense.weight', tensor([[-0.0405,  0.0720,  0.0527,  ...,  0.0654,  0.0432, -0.0180],\n",
      "        [ 0.0901, -0.0525, -0.0628,  ...,  0.1002, -0.0536, -0.0246],\n",
      "        [ 0.0206, -0.0817,  0.1335,  ...,  0.0551,  0.2007, -0.0160],\n",
      "        ...,\n",
      "        [-0.0943,  0.0220,  0.0583,  ...,  0.0758,  0.0526, -0.0250],\n",
      "        [ 0.0854,  0.0728,  0.0494,  ...,  0.1059,  0.0743, -0.0677],\n",
      "        [-0.0909,  0.0170, -0.0689,  ..., -0.0627,  0.0196,  0.0043]])), ('bert.encoder.layer.3.intermediate.dense.bias', tensor([-0.1103, -0.0719, -0.0499,  ..., -0.0455, -0.1266,  0.0422])), ('bert.encoder.layer.3.output.dense.weight', tensor([[-0.1142, -0.0884, -0.0204,  ..., -0.0025, -0.0964,  0.0594],\n",
      "        [ 0.0707, -0.1604, -0.0630,  ..., -0.0839,  0.0556, -0.0989],\n",
      "        [ 0.0419,  0.0780, -0.0159,  ..., -0.0591,  0.0202,  0.0844],\n",
      "        ...,\n",
      "        [-0.0692,  0.0484,  0.0153,  ..., -0.0080,  0.0106,  0.0110],\n",
      "        [-0.0161,  0.0209, -0.0318,  ...,  0.0720,  0.0545, -0.0280],\n",
      "        [-0.0500,  0.1009,  0.0320,  ..., -0.0105, -0.0604,  0.0040]])), ('bert.encoder.layer.3.output.dense.bias', tensor([ 4.8630e-02,  3.2228e-02,  2.0358e-03,  3.9173e-02,  1.0539e-02,\n",
      "        -5.9681e-02, -4.0190e-02,  6.0075e-02, -6.8565e-02,  6.0475e-04,\n",
      "         7.7717e-02, -4.1091e-04, -6.8828e-03,  2.5351e-02, -3.7063e-02,\n",
      "         4.4218e-03,  1.2283e-02, -3.9212e-04,  3.2663e-02, -1.9174e-02,\n",
      "         9.7742e-03,  7.0689e-02,  1.5140e-02,  1.2436e-02,  3.6726e-02,\n",
      "         4.6557e-02, -5.8118e-02, -1.4190e-02, -3.4095e-03,  1.4215e-02,\n",
      "        -2.0959e-02,  4.4220e-02, -6.5596e-02, -6.0913e-02, -4.0027e-03,\n",
      "         4.5134e-02,  2.4933e-02,  9.6692e-03, -5.0294e-02, -3.1695e-02,\n",
      "        -1.2701e-02, -1.7277e-02,  1.0439e-02, -7.6971e-02, -8.8591e-02,\n",
      "         7.0034e-02,  1.7308e-02,  7.8292e-03,  6.3406e-02,  4.3618e-02,\n",
      "         6.2807e-02, -2.1979e-02,  2.2041e-02,  3.0632e-02, -1.7499e-02,\n",
      "         6.1791e-02, -1.8968e-02,  1.4783e-02, -6.3361e-02, -1.0014e-03,\n",
      "        -1.1925e-02,  1.6888e-02,  1.3619e-02,  3.0306e-02,  2.9926e-02,\n",
      "         9.9301e-03, -2.0458e-03,  6.7287e-03, -3.0178e-02,  1.0022e-02,\n",
      "         6.2951e-03,  5.4141e-03, -4.4647e-02,  1.3271e-02,  2.8742e-03,\n",
      "        -1.3361e-02, -2.9446e-02,  2.2340e-02,  1.6043e-02, -3.3673e-02,\n",
      "        -1.7479e-02,  3.2383e-02,  5.0902e-02,  2.8784e-03,  9.0870e-03,\n",
      "        -5.2726e-03, -1.9399e-02,  2.4547e-02, -2.8077e-02, -4.5043e-03,\n",
      "         2.0159e-02, -2.3857e-02, -6.3390e-02,  3.9810e-02,  4.2953e-02,\n",
      "        -1.7261e-02,  1.0862e-02, -1.1424e-02, -4.7260e-02,  2.2913e-03,\n",
      "        -8.9644e-03,  1.3588e-02, -1.1681e-04,  3.8049e-02, -2.6475e-02,\n",
      "         3.1241e-02,  3.2589e-02,  5.1907e-02, -2.1802e-02, -1.6821e-02,\n",
      "         4.5697e-02,  3.7421e-03, -4.0006e-02,  3.7217e-02, -9.5896e-03,\n",
      "        -1.3695e-03, -2.8940e-03,  6.1423e-03,  4.0211e-02, -8.1247e-03,\n",
      "        -1.7497e-02, -2.0919e-02,  1.4163e-02,  3.0832e-02,  2.9020e-02,\n",
      "        -1.3588e-03, -1.9187e-02, -1.6005e-02,  2.9397e-02, -1.1980e-02,\n",
      "        -7.6095e-03,  1.4390e-02, -6.1149e-03, -9.2242e-03, -2.1871e-02,\n",
      "         2.5212e-02,  1.1571e-02,  2.5870e-03, -1.6822e-02, -5.3763e-03,\n",
      "        -1.0860e-02, -1.7767e-02, -1.1947e-02, -1.6390e-02, -6.1049e-03,\n",
      "         2.3346e-02,  8.7632e-03, -1.8775e-02, -3.0072e-02, -1.5542e-02,\n",
      "        -4.5756e-03,  1.6920e-02,  1.4539e-02, -5.0420e-02,  3.3935e-02,\n",
      "        -6.9304e-02, -1.8693e-02,  3.7301e-02,  6.8490e-05,  4.1243e-03,\n",
      "         5.4425e-03,  3.0878e-02, -4.1894e-03,  1.3999e-02, -6.9612e-03,\n",
      "        -1.0744e-02,  3.9714e-02, -2.0778e-02, -7.2043e-03,  3.6861e-02,\n",
      "         1.9402e-03,  4.5247e-03,  4.8877e-03, -2.2437e-02, -1.9973e-03,\n",
      "         3.1353e-02, -1.6331e-02,  4.0801e-02, -3.2848e-02, -4.7942e-03,\n",
      "        -9.2674e-03,  4.6384e-02, -1.0150e-02,  2.8941e-02, -2.2000e-02,\n",
      "         2.1087e-02,  1.0413e-03,  3.9920e-03, -2.4380e-02, -9.7461e-03,\n",
      "        -4.5331e-02, -3.8338e-02, -6.1522e-02, -4.6056e-02, -4.0727e-02,\n",
      "         2.9136e-02,  1.0465e-02,  2.7672e-03, -3.6580e-02, -4.3492e-02,\n",
      "         7.0431e-03, -2.2122e-02, -6.9756e-03,  3.9889e-02, -2.9048e-02,\n",
      "        -1.5247e-02, -8.8149e-03, -8.4156e-04,  1.0532e-02,  3.1809e-03,\n",
      "         4.9357e-02, -4.4346e-04,  1.4355e-02,  3.4130e-02, -1.0025e-02,\n",
      "         1.0955e-02, -4.0167e-02, -2.9244e-02,  1.3125e-02,  2.8446e-02,\n",
      "        -4.3021e-02, -2.5038e-02, -1.7375e-01, -4.4489e-03,  2.2855e-02,\n",
      "        -4.0013e-02,  5.6243e-02, -7.6474e-03, -5.9032e-03,  1.5434e-02,\n",
      "         3.9620e-02, -7.7618e-03, -5.4451e-03, -2.1939e-03,  5.5174e-03,\n",
      "        -8.0229e-03, -1.4968e-02,  3.2994e-02,  1.1202e-02, -3.0948e-02,\n",
      "        -2.3777e-03,  1.7319e-02,  1.6122e-02,  1.1171e-02,  1.4979e-02,\n",
      "        -9.7656e-03,  2.0266e-02, -1.4707e-02, -1.6106e-02,  3.5887e-02,\n",
      "        -2.1823e-02, -1.0998e-02, -6.3521e-03,  3.2410e-02, -1.6817e-02,\n",
      "         1.8203e-02,  1.1613e-02,  3.1893e-02, -5.4959e-02,  4.3676e-03,\n",
      "         4.0792e-02, -6.5596e-04, -5.5368e-03,  1.2287e-02, -1.7374e-02,\n",
      "        -1.8762e-03, -8.0679e-03, -2.5863e-02,  4.4093e-02,  3.0618e-03,\n",
      "        -3.9911e-02, -3.0368e-03,  5.3770e-03,  3.8357e-02,  2.7770e-02,\n",
      "         1.7312e-01, -8.6265e-03,  8.8256e-03, -2.2075e-02,  1.4042e-02,\n",
      "         1.3990e-02, -2.2252e-03,  2.0393e-02,  1.4780e-02,  1.4907e-02,\n",
      "        -1.3698e-02, -3.0494e-02,  3.8361e-02, -1.5598e-02, -5.2890e-02,\n",
      "        -5.3601e-02,  9.6322e-03,  8.2911e-03, -9.4598e-03,  2.5951e-02,\n",
      "        -3.3956e-02,  1.1304e-02, -1.5834e-02, -4.2956e-02,  1.9646e-02,\n",
      "        -9.8766e-02, -6.2295e-02,  1.8974e-02, -1.7742e-02, -7.8803e-03,\n",
      "        -1.0296e-03,  2.4644e-02,  1.5846e-02,  4.8057e-02, -1.8866e-02,\n",
      "        -1.1448e-02,  2.6800e-02, -2.5362e-02, -4.1111e-03, -1.5034e-02,\n",
      "        -1.5318e-02,  1.0863e-02,  1.2274e-02,  1.7318e-02, -2.1159e-03,\n",
      "         2.8544e-02,  9.1807e-03,  9.3025e-03,  1.7564e-02,  1.6876e-02,\n",
      "         1.8967e-02, -2.0316e-02, -2.0457e-02, -1.1405e-02, -1.3410e-03,\n",
      "        -1.2360e-02,  1.7188e-02, -7.0907e-03,  4.4246e-02, -1.0703e-02,\n",
      "         5.5035e-03,  8.0534e-03,  8.0571e-03, -4.1319e-02,  2.6935e-02,\n",
      "         1.7057e-02,  3.1906e-02,  4.4481e-02,  1.9586e-02, -3.3808e-02,\n",
      "         1.3627e-02, -6.3812e-03, -3.7642e-02, -3.9676e-02,  2.1603e-02,\n",
      "        -3.3334e-02,  2.5666e-02, -5.7497e-02,  1.9274e-02, -2.4879e-02,\n",
      "         1.1169e-02, -4.4062e-02,  2.4471e-02,  1.2548e-03,  2.1147e-02,\n",
      "        -1.2546e-02, -1.8196e-02,  3.2020e-03,  1.3220e-02,  4.5821e-02,\n",
      "        -6.7199e-02, -4.4966e-02,  4.3560e-03, -3.8316e-02, -1.4453e-02,\n",
      "        -1.6415e-02,  1.3461e-03,  4.1207e-02,  8.6680e-04,  2.0869e-02,\n",
      "         1.5354e-02, -4.6911e-03, -3.3390e-02, -8.8124e-03, -2.4068e-03,\n",
      "        -2.6034e-02, -1.0885e-02, -1.8853e-02,  5.1000e-03])), ('bert.encoder.layer.3.output.LayerNorm.weight', tensor([0.8228, 0.7782, 0.8334, 0.8491, 0.6597, 0.7776, 0.7825, 0.7565, 0.7239,\n",
      "        0.8068, 0.7851, 0.7632, 0.7784, 0.7901, 0.7804, 0.7859, 0.8054, 0.7831,\n",
      "        0.7743, 0.8288, 0.7782, 0.7319, 0.8010, 0.7765, 0.7900, 0.7584, 0.7120,\n",
      "        0.7852, 0.7941, 0.7865, 0.8019, 0.8179, 0.7885, 0.7940, 0.7763, 0.8304,\n",
      "        0.7197, 0.7902, 0.7666, 0.7912, 0.7636, 0.7548, 0.8344, 0.7803, 0.8163,\n",
      "        0.8114, 0.8032, 0.8490, 0.7861, 0.8116, 0.8273, 0.8067, 0.7903, 0.8277,\n",
      "        0.8432, 0.8227, 0.8196, 0.8680, 0.8036, 0.8546, 0.8592, 0.8756, 0.8796,\n",
      "        0.8983, 0.8789, 0.9260, 0.8726, 0.9026, 0.8657, 0.8886, 0.8901, 0.8424,\n",
      "        0.8801, 0.8935, 0.7114, 0.9034, 0.8315, 0.9076, 0.8954, 0.8640, 0.7254,\n",
      "        0.8223, 0.8577, 0.9215, 0.9185, 0.8522, 0.9265, 0.8813, 0.8858, 0.7782,\n",
      "        0.9151, 0.8625, 0.9175, 0.8322, 0.8400, 0.8719, 0.9433, 0.9075, 0.8757,\n",
      "        0.9153, 0.9249, 0.8374, 0.9161, 0.9127, 0.8403, 0.8611, 0.8935, 0.8858,\n",
      "        0.9542, 0.8893, 0.9016, 0.9044, 0.8708, 0.8596, 0.9062, 0.8970, 0.9051,\n",
      "        0.8601, 0.9177, 0.9197, 0.6509, 0.9280, 0.9245, 0.9210, 0.9099, 0.8902,\n",
      "        0.9292, 0.8755, 0.8937, 0.8813, 0.8154, 0.8866, 0.9186, 0.9339, 0.9188,\n",
      "        0.9348, 0.9213, 0.9086, 0.8986, 0.9016, 0.9161, 0.8439, 0.9323, 0.8774,\n",
      "        0.9156, 0.9146, 0.9119, 0.9168, 0.8965, 0.9111, 0.9252, 0.9066, 0.9015,\n",
      "        0.9512, 0.8691, 0.9050, 0.8904, 0.8709, 0.8795, 0.8974, 0.9070, 0.9181,\n",
      "        0.9054, 0.9052, 0.9193, 0.8736, 0.9140, 0.9097, 0.8893, 0.9016, 0.9289,\n",
      "        0.8904, 0.9253, 0.9164, 0.9330, 0.9550, 0.9348, 0.9097, 0.9198, 0.8268,\n",
      "        0.9231, 0.8187, 0.9134, 0.8872, 0.8394, 0.8994, 0.9136, 1.2327, 0.9012,\n",
      "        0.8768, 0.9274, 0.8921, 0.8877, 0.9193, 0.8602, 0.8942, 0.9095, 0.9148,\n",
      "        0.9114, 0.9013, 0.8622, 0.8824, 0.8705, 0.9360, 0.8942, 0.9116, 0.8890,\n",
      "        0.8932, 0.7466, 0.9110, 0.8925, 0.9248, 0.8860, 0.9128, 0.9242, 0.9009,\n",
      "        0.9221, 0.8932, 0.8903, 0.9147, 0.8725, 0.9050, 0.7235, 0.8908, 0.8109,\n",
      "        0.9350, 0.8961, 0.8699, 0.8923, 0.8908, 0.8615, 0.9184, 0.8853, 0.9137,\n",
      "        0.8595, 0.9452, 0.8920, 0.9308, 0.8951, 0.9410, 0.8937, 0.8933, 0.9018,\n",
      "        0.9020, 0.9164, 0.8660, 0.8877, 0.8904, 0.9206, 0.9108, 0.9057, 0.8762,\n",
      "        0.9216, 0.7917, 0.9338, 0.8815, 0.9273, 0.9363, 0.7985, 0.9264, 0.9450,\n",
      "        0.8943, 0.8863, 0.9039, 0.9137, 0.8817, 0.9127, 0.9312, 0.8807, 0.9152,\n",
      "        0.9028, 0.8057, 0.8647, 0.8955, 0.8961, 0.7942, 0.9169, 0.8865, 0.9162,\n",
      "        0.8203, 0.9029, 0.8810, 0.8991, 0.8944, 0.8766, 0.9039, 0.8915, 0.8103,\n",
      "        0.9143, 0.9126, 0.9046, 0.9170, 0.9042, 0.8509, 0.9081, 0.8909, 0.9214,\n",
      "        0.8989, 0.8312, 0.8398, 0.7473, 0.8977, 0.9186, 0.9066, 0.9223, 0.8835,\n",
      "        0.9028, 0.9108, 0.9084, 0.9118, 0.8938, 0.9148, 0.9198, 0.9278, 0.8912,\n",
      "        0.8962, 0.8855, 0.8753, 0.8862, 0.9271, 0.9375, 0.8953, 0.9283, 0.8950,\n",
      "        0.9017, 0.8722, 0.8743, 0.8945, 0.9003, 0.9013, 0.8763, 0.8834, 0.8887,\n",
      "        0.8140, 0.9110, 0.8011, 0.9108, 0.9034, 0.8939, 0.9147, 0.9375, 0.9319,\n",
      "        0.8883, 0.8751, 0.8773, 0.9267, 0.9151, 0.8773, 0.7455, 0.8899, 0.8144,\n",
      "        0.9078, 0.8941, 0.8987, 0.8983, 0.8726, 0.8658, 0.9002, 0.9187, 0.8420,\n",
      "        0.8772, 0.9032, 0.9083, 0.8522, 0.7537, 0.8367, 0.8951, 0.8890, 0.9014,\n",
      "        0.8390, 0.8939, 0.8954, 0.9139, 0.9163, 0.9235, 0.8745, 0.8763, 0.8723,\n",
      "        0.8572, 0.8842, 0.8726, 0.8959, 0.9094, 0.8812])), ('bert.encoder.layer.3.output.LayerNorm.bias', tensor([ 3.2220e-02,  7.5735e-02,  2.3972e-02,  3.0039e-02, -2.4720e-02,\n",
      "         2.6531e-02, -4.1614e-02,  4.1851e-02, -5.5341e-02,  1.6778e-02,\n",
      "         5.1517e-02,  5.1401e-02, -2.5121e-02,  4.5467e-02, -3.2970e-02,\n",
      "        -1.0919e-02,  5.1895e-02,  1.7936e-02,  2.5095e-02, -2.0583e-02,\n",
      "         3.4685e-02, -1.8892e-02,  1.1137e-01, -5.1779e-02,  2.2123e-02,\n",
      "        -1.3126e-03, -5.7865e-02, -1.1607e-02,  4.2363e-03,  1.0544e-01,\n",
      "         4.0609e-02,  1.7177e-02,  1.9023e-02, -4.9459e-03,  3.9099e-02,\n",
      "         3.5666e-02, -5.1336e-02,  1.1920e-02,  2.5627e-02, -5.8192e-02,\n",
      "         3.1805e-02, -1.6541e-02, -1.2855e-02,  1.3225e-02, -5.9430e-02,\n",
      "         3.8740e-02,  1.0319e-01, -5.4904e-02,  1.2804e-01, -7.9240e-02,\n",
      "         2.0671e-02, -7.8944e-02, -7.4584e-02,  2.2554e-03,  2.6381e-02,\n",
      "         5.1719e-02,  2.1153e-02,  9.7985e-02, -5.3694e-03,  2.8286e-02,\n",
      "         1.2922e-02,  1.0264e-01, -4.1801e-02,  2.4258e-02, -2.5064e-02,\n",
      "         7.9638e-02, -4.3695e-03, -2.3832e-03, -1.0667e-01,  1.3914e-03,\n",
      "         4.5082e-03,  1.2571e-02, -1.4074e-02, -8.3212e-03,  2.6434e-01,\n",
      "        -6.4731e-02, -1.0225e-01,  3.3523e-02,  7.8244e-02,  9.0602e-02,\n",
      "        -2.2902e-01, -1.5122e-01,  1.3433e-02, -6.9862e-03,  1.7704e-02,\n",
      "         1.0321e-01, -5.8430e-02, -1.2017e-01,  7.3693e-02, -1.5949e-01,\n",
      "         1.4853e-02,  7.8283e-02,  8.1644e-03, -4.2932e-02,  3.1079e-03,\n",
      "        -5.0604e-02,  4.0301e-02, -1.4101e-02,  5.2390e-02, -2.0469e-02,\n",
      "         4.6051e-02, -8.4282e-02,  2.0322e-02,  3.9370e-02,  1.1057e-01,\n",
      "        -5.7946e-02,  5.3361e-02, -1.6914e-02, -3.6094e-03, -5.2937e-02,\n",
      "        -1.3735e-02, -5.3514e-02,  7.5227e-02, -8.0563e-02, -2.3850e-02,\n",
      "         3.1760e-02, -2.3595e-02,  9.8451e-02,  7.4120e-02, -8.5694e-02,\n",
      "        -2.2756e-01, -2.6131e-02, -5.4150e-02, -4.2141e-03, -3.3445e-02,\n",
      "         8.4593e-02,  7.4602e-03,  3.3482e-02,  4.6351e-02, -4.1380e-02,\n",
      "        -9.2577e-02, -4.4779e-02,  3.7328e-02,  3.0968e-02,  1.9508e-02,\n",
      "         2.4418e-02,  1.2518e-02,  3.1194e-02,  6.2475e-02,  1.6581e-02,\n",
      "         8.6029e-02, -8.1053e-02,  3.0287e-02,  1.0443e-02,  5.4673e-02,\n",
      "         2.0542e-02,  4.2653e-02,  2.0098e-02, -3.9819e-02, -4.3890e-02,\n",
      "        -2.7629e-02, -3.9345e-02,  2.0980e-02,  7.0837e-03, -1.8132e-02,\n",
      "        -2.3388e-02,  2.7572e-02, -4.4657e-02, -5.4802e-02, -5.1264e-02,\n",
      "         2.4821e-02, -7.0998e-03,  5.1437e-02, -1.1553e-01,  6.5626e-03,\n",
      "        -6.9382e-02,  6.0387e-02, -6.3764e-02,  4.8328e-04,  8.2100e-03,\n",
      "         3.0749e-02,  4.5537e-02,  4.0208e-02, -8.1391e-02,  8.7309e-02,\n",
      "        -1.3089e-03,  2.1733e-02,  1.7177e-02, -2.1338e-02, -1.1335e-01,\n",
      "         3.3383e-02, -9.0348e-02,  1.0044e-02, -7.6383e-03,  1.2908e-01,\n",
      "         8.7345e-02,  8.7858e-02, -2.5001e-02, -3.2933e-02, -7.7528e-02,\n",
      "         7.5692e-03,  2.3581e-02,  4.0217e-02, -5.2499e-02,  1.0615e-01,\n",
      "        -5.6384e-02, -4.1312e-02, -3.4353e-02, -2.8788e-02, -4.6773e-02,\n",
      "         7.7488e-02,  2.8835e-02,  1.1010e-02, -2.9185e-02,  6.4635e-02,\n",
      "        -2.7772e-02, -1.1913e-02, -6.1713e-02,  2.4414e-01, -1.7247e-02,\n",
      "         5.9345e-02, -7.3474e-05,  3.7368e-02, -2.1914e-02,  4.7045e-02,\n",
      "        -5.3417e-02,  1.0062e-02, -3.4221e-03,  5.1383e-02, -2.7405e-02,\n",
      "         7.7097e-02,  4.1445e-02, -6.4032e-02,  3.9947e-03,  1.3273e-01,\n",
      "        -6.3322e-02,  3.7025e-03, -1.1567e-01, -2.6024e-02, -1.8701e-02,\n",
      "         1.5644e-01, -6.6894e-03,  1.1236e-01, -1.2507e-02,  6.7712e-02,\n",
      "         1.1100e-02,  4.7335e-02,  6.2494e-02,  9.1731e-02, -3.5235e-02,\n",
      "         2.4362e-02, -5.6761e-02,  3.0935e-02, -5.2626e-02, -4.5137e-02,\n",
      "        -6.4594e-02,  1.1228e-01,  5.5438e-03,  5.7604e-02,  5.8286e-03,\n",
      "        -1.4994e-02, -2.0958e-02,  3.9707e-02,  1.5658e-01, -9.4258e-03,\n",
      "        -6.5897e-02, -6.7427e-04,  1.5066e-02,  1.1546e-01,  9.5040e-03,\n",
      "         2.5861e-02, -4.0162e-02,  1.6336e-02,  4.1558e-03, -5.0799e-02,\n",
      "        -6.4696e-02, -4.5854e-02, -6.8723e-02,  7.5843e-02, -4.8800e-02,\n",
      "         7.3887e-03, -1.7918e-01,  8.6824e-02, -3.8340e-02,  7.9443e-02,\n",
      "         4.3871e-02,  2.1159e-02, -5.1123e-02, -3.9479e-02, -1.3535e-01,\n",
      "         7.5730e-02, -3.9917e-03,  3.6996e-02,  7.9846e-03,  6.2941e-02,\n",
      "        -3.6776e-02,  1.7156e-02, -6.2080e-02,  2.5256e-02, -1.8236e-02,\n",
      "         4.2947e-02, -4.1730e-02,  4.7502e-02,  3.2429e-02,  5.2694e-02,\n",
      "        -7.6203e-02, -1.1336e-02, -3.4099e-03,  9.9853e-02, -9.3655e-02,\n",
      "         4.0278e-02, -1.9919e-03, -4.4935e-03,  8.8519e-03,  3.3604e-02,\n",
      "        -3.5582e-02,  1.2396e-01, -5.5826e-02,  2.3825e-02, -1.6795e-02,\n",
      "         7.9495e-02, -1.1336e-02, -6.1001e-02,  5.2651e-03,  2.3104e-02,\n",
      "         5.2691e-03,  1.0853e-01, -5.5674e-02, -8.0097e-02, -5.9617e-03,\n",
      "        -4.3182e-02,  7.4823e-02,  6.9804e-02, -9.0592e-02,  5.4430e-02,\n",
      "        -4.6477e-02,  5.6426e-02,  4.8890e-02, -6.4499e-03, -4.5797e-02,\n",
      "         9.1127e-02, -1.0984e-01,  3.3511e-02, -1.3295e-01,  1.3161e-02,\n",
      "        -5.3419e-02,  2.2288e-02, -3.5086e-02, -6.9442e-02, -4.9085e-02,\n",
      "        -1.4917e-02, -1.0269e-02,  5.6077e-02, -8.4674e-02,  1.5308e-01,\n",
      "         4.8869e-02,  4.8292e-02,  1.0999e-02,  2.0384e-01, -6.2000e-02,\n",
      "         1.7217e-01, -4.1541e-02,  1.7589e-02,  9.3752e-03, -4.5286e-02,\n",
      "        -9.8598e-02,  4.6331e-02, -5.7924e-02,  5.3698e-02, -6.8053e-02,\n",
      "         1.1054e-02, -2.2203e-02,  5.2105e-02, -1.9409e-02, -6.4636e-02,\n",
      "         2.3235e-02, -6.0054e-02, -5.6637e-02, -9.5078e-03, -1.3553e-02,\n",
      "         5.8234e-02,  5.7574e-02,  3.5889e-02, -3.6901e-03, -2.3929e-02,\n",
      "         4.7714e-02, -6.0038e-03, -3.1054e-02,  7.2313e-02, -4.3023e-02,\n",
      "        -6.6228e-02,  3.2573e-02,  4.9385e-02, -4.7279e-02])), ('bert.encoder.layer.4.attention.self.query.weight', tensor([[ 0.0134, -0.0003,  0.0296,  ...,  0.0841, -0.0115,  0.0242],\n",
      "        [ 0.0028, -0.0050,  0.0093,  ...,  0.0153,  0.0012,  0.0718],\n",
      "        [ 0.0770, -0.1267, -0.0128,  ...,  0.0722,  0.0666, -0.0912],\n",
      "        ...,\n",
      "        [-0.0068, -0.0471,  0.0561,  ...,  0.0061, -0.0098, -0.0295],\n",
      "        [ 0.0193, -0.0407, -0.0124,  ...,  0.0365, -0.0440, -0.0670],\n",
      "        [ 0.0287, -0.0325,  0.0393,  ..., -0.0636, -0.0154,  0.0997]])), ('bert.encoder.layer.4.attention.self.query.bias', tensor([ 0.0619, -0.0372,  0.0152,  0.0681, -0.0067,  0.0112,  0.0281, -0.0385,\n",
      "        -0.0222, -0.0261,  0.0251, -0.0803, -0.0417,  0.0029, -0.0560, -0.0570,\n",
      "         0.0405,  0.0191, -0.0226,  0.0177,  0.0228, -0.0783, -0.0128, -0.0339,\n",
      "        -0.0027,  0.0387, -0.0166, -0.0093,  0.0696,  0.0058, -0.0504,  0.0280,\n",
      "         0.1001,  0.0305,  0.0092, -0.0010, -0.0478,  0.0057, -0.0221,  0.0471,\n",
      "        -0.1063, -0.0623,  0.0128,  0.0059,  0.0394,  0.0413, -0.0629, -0.0287,\n",
      "        -0.0734,  0.0318,  0.0117, -0.0617, -0.0989, -0.0324, -0.0834, -0.1016,\n",
      "         0.0467, -0.0318,  0.0271,  0.0599, -0.0581,  0.1673, -0.0473, -0.0284,\n",
      "         0.1050,  0.0557, -0.0713,  0.0383, -0.0309, -0.0455, -0.0603, -0.0737,\n",
      "        -0.0495,  0.1000,  0.0344, -0.0450, -0.0490,  0.0217,  0.1296, -0.0632,\n",
      "         0.0207,  0.0705, -0.0075,  0.1256,  0.0447,  0.0334,  0.1158, -0.0473,\n",
      "         0.0128, -0.0092, -0.0584,  0.0009, -0.0148, -0.1159, -0.0822,  0.0092,\n",
      "        -0.0386, -0.0116, -0.0163,  0.0375, -0.0328,  0.0009,  0.0388,  0.0460,\n",
      "        -0.0040,  0.0487,  0.0447,  0.0018,  0.0243,  0.0146, -0.0419,  0.0312,\n",
      "        -0.0416, -0.0381,  0.0028,  0.0029, -0.0203, -0.0008,  0.0036,  0.0029,\n",
      "         0.0540,  0.0148,  0.0345,  0.0150,  0.0152, -0.0056,  0.0486, -0.0206,\n",
      "         0.0088,  0.0403,  0.0564, -0.0008,  0.0242,  0.0118, -0.0352, -0.0015,\n",
      "        -0.0046, -0.0276,  0.0118,  0.0140, -0.0162, -0.0077, -0.0067, -0.0046,\n",
      "        -0.0169, -0.0012,  0.0125, -0.0251, -0.0207,  0.0064,  0.0076,  0.0134,\n",
      "        -0.0080,  0.0287, -0.0171, -0.0264,  0.0124,  0.0408,  0.0135,  0.0216,\n",
      "        -0.0097,  0.0054, -0.0051,  0.0219, -0.0055,  0.0128,  0.0181, -0.0185,\n",
      "        -0.0019, -0.0181,  0.0255,  0.0232, -0.0177, -0.0078,  0.0254,  0.0053,\n",
      "        -0.0105, -0.0135, -0.0301, -0.0167, -0.0108, -0.0007,  0.0312, -0.0383,\n",
      "         0.0181,  0.0445, -0.0101,  0.0046, -0.0007,  0.0368,  0.0417, -0.0281,\n",
      "        -0.0305, -0.0276,  0.1521,  0.0416, -0.0461, -0.1568, -0.0536, -0.0529,\n",
      "        -0.0286, -0.0169, -0.0536,  0.1770, -0.1482,  0.0179,  0.0716, -0.0293,\n",
      "        -0.1257,  0.0333,  0.0135,  0.0499, -0.1608,  0.0259,  0.0091, -0.0349,\n",
      "        -0.0623, -0.0431, -0.1268, -0.0895, -0.0035,  0.0528,  0.1180,  0.0350,\n",
      "        -0.0971, -0.0150,  0.0253,  0.0171, -0.0218, -0.0299,  0.0178,  0.0129,\n",
      "        -0.0078, -0.1150,  0.0291, -0.0545,  0.0093, -0.0365,  0.0430,  0.0303,\n",
      "        -0.0554, -0.0536, -0.0454,  0.0152, -0.0109, -0.0749,  0.0157,  0.0286,\n",
      "         0.0764,  0.0094,  0.0852,  0.0388,  0.0201, -0.0777, -0.0554,  0.0415,\n",
      "        -0.0482,  0.0172,  0.0692,  0.0322, -0.0095,  0.0120, -0.0048, -0.0405,\n",
      "        -0.0940, -0.0218, -0.0830, -0.0275,  0.0825, -0.1128, -0.0111,  0.0423,\n",
      "         0.0612,  0.0256,  0.1331, -0.0063, -0.0481,  0.0514,  0.0182, -0.0251,\n",
      "        -0.0217, -0.0034,  0.1377,  0.0061,  0.0460, -0.0055, -0.0370, -0.0582,\n",
      "         0.0404,  0.0061,  0.0018,  0.0314,  0.0007,  0.0148,  0.0215, -0.0221,\n",
      "         0.0184,  0.0165, -0.0091, -0.0475,  0.0105, -0.0137, -0.0037,  0.0039,\n",
      "        -0.0142, -0.0145,  0.0162, -0.0358,  0.0204,  0.0128,  0.0738, -0.0150,\n",
      "        -0.0210,  0.0247,  0.0193,  0.0462, -0.0027,  0.0067,  0.0011, -0.0139,\n",
      "        -0.0714, -0.0180,  0.0620, -0.0063, -0.0028, -0.1472,  0.0654, -0.0915,\n",
      "        -0.0611, -0.0011, -0.0509, -0.0781,  0.1475,  0.1011, -0.0194,  0.0730,\n",
      "        -0.0177,  0.0575, -0.0127, -0.0250,  0.0410,  0.0833, -0.0057, -0.0893,\n",
      "         0.0490,  0.0843, -0.0238,  0.0382, -0.0164, -0.1047,  0.0180,  0.0173,\n",
      "        -0.0619, -0.0161,  0.0021,  0.0743,  0.0758, -0.0515,  0.0045, -0.0720,\n",
      "        -0.0084, -0.0521,  0.0957, -0.0043, -0.0570, -0.0137,  0.0076, -0.0151,\n",
      "         0.0355, -0.0289, -0.0647,  0.0552, -0.0630,  0.0181, -0.0659,  0.0396,\n",
      "         0.0367,  0.0300, -0.0150,  0.0217,  0.0413, -0.0082,  0.0131,  0.0035])), ('bert.encoder.layer.4.attention.self.key.weight', tensor([[-0.0203,  0.0030, -0.1160,  ...,  0.0195, -0.0256, -0.0733],\n",
      "        [ 0.0448,  0.1160,  0.1059,  ..., -0.0247,  0.0098,  0.0486],\n",
      "        [ 0.1030, -0.0941,  0.1090,  ...,  0.0128,  0.0316, -0.0820],\n",
      "        ...,\n",
      "        [ 0.0470, -0.1049,  0.0408,  ...,  0.0096,  0.0027,  0.0452],\n",
      "        [ 0.0092, -0.0988,  0.0034,  ...,  0.0214, -0.0885, -0.1089],\n",
      "        [ 0.0437, -0.0253,  0.0584,  ..., -0.0948,  0.0404,  0.0497]])), ('bert.encoder.layer.4.attention.self.key.bias', tensor([-2.3200e-03,  3.9112e-05, -1.5141e-03,  4.6977e-04,  2.3327e-03,\n",
      "         1.0734e-02,  8.6768e-04, -2.9015e-03, -1.4343e-03, -2.8127e-04,\n",
      "         1.5471e-03, -1.3799e-03, -6.8960e-03,  9.0867e-03, -2.6435e-03,\n",
      "        -1.8328e-03,  3.1847e-03,  3.1067e-03, -3.0348e-03,  2.4816e-03,\n",
      "         3.5063e-03, -6.7523e-03, -1.1306e-03, -4.5409e-03, -1.0314e-03,\n",
      "        -4.3751e-03, -3.8312e-04, -3.4800e-03,  1.9754e-03, -1.4763e-03,\n",
      "        -3.5102e-03, -1.2375e-03, -1.5220e-03, -4.4959e-04,  1.1735e-03,\n",
      "         1.3828e-03, -2.4246e-03,  1.0003e-03, -3.8062e-04, -9.8256e-04,\n",
      "         9.1326e-04,  2.4217e-04,  2.4113e-03,  7.8169e-04, -2.0737e-03,\n",
      "         3.5444e-04,  3.9676e-04, -4.6769e-03,  1.5974e-04, -1.6749e-03,\n",
      "        -1.2480e-03,  2.8482e-03,  3.9654e-03,  2.4258e-03, -1.7492e-03,\n",
      "        -3.9779e-03,  4.7958e-04,  1.1581e-03,  4.0337e-03, -1.0744e-03,\n",
      "         4.8202e-04,  5.8140e-05, -2.2334e-03, -2.4426e-03,  2.8932e-02,\n",
      "         7.6024e-03, -1.8305e-02,  1.5811e-02, -2.1624e-02,  1.6615e-03,\n",
      "        -2.1057e-02, -1.4328e-03, -5.9425e-03, -1.1158e-02, -1.8191e-02,\n",
      "        -1.6956e-02,  4.2924e-03, -4.3814e-03,  3.2395e-02,  2.5543e-02,\n",
      "         1.7763e-02,  2.7796e-02, -2.9264e-02,  3.8374e-02,  1.6534e-02,\n",
      "        -6.9722e-03, -2.7700e-04, -1.2540e-02,  1.2100e-02,  8.5475e-03,\n",
      "         4.1304e-03, -1.2252e-02,  1.2923e-02, -2.6934e-02, -1.1950e-02,\n",
      "         1.3531e-02,  6.0295e-04,  1.2019e-04,  3.8215e-04, -1.8797e-03,\n",
      "        -3.2997e-04, -5.4281e-04, -5.3635e-04, -2.7238e-03,  4.2531e-04,\n",
      "        -1.0568e-03, -4.0436e-04, -1.3453e-03,  5.7677e-04,  1.1221e-03,\n",
      "         2.1373e-04,  3.1358e-03,  1.8878e-03, -4.9693e-03,  1.4102e-03,\n",
      "        -1.5908e-03,  1.9889e-03,  2.0046e-03, -2.1208e-04, -9.9832e-04,\n",
      "        -2.4614e-03,  1.5016e-03, -9.6710e-04, -6.8632e-04, -9.3250e-04,\n",
      "         3.4214e-03, -2.4134e-03,  9.9591e-04,  5.1252e-03, -1.6064e-04,\n",
      "         2.2098e-03,  1.2904e-03,  1.6085e-03, -2.2132e-03, -2.2871e-03,\n",
      "        -6.1307e-04, -3.2810e-03,  7.9160e-05, -1.7498e-03, -2.5869e-03,\n",
      "         5.7149e-03, -1.3281e-03, -1.5082e-03,  2.5445e-03,  3.0699e-04,\n",
      "        -3.4172e-03,  5.0121e-04, -5.8104e-04,  1.9587e-03, -9.3810e-05,\n",
      "        -8.0884e-04, -8.0811e-05, -7.8896e-04,  1.0321e-03, -1.9536e-03,\n",
      "         3.1937e-03, -3.5952e-03,  3.9234e-04, -8.3327e-04,  3.2430e-03,\n",
      "        -2.8484e-04,  7.4454e-04, -2.3717e-03,  1.4608e-03,  6.8636e-04,\n",
      "        -6.5597e-04,  1.7668e-03,  4.9426e-03,  4.2106e-04,  1.1749e-03,\n",
      "        -5.9693e-05, -1.5552e-03,  6.9537e-04, -1.0861e-04,  4.1892e-04,\n",
      "         9.0410e-04,  2.3427e-03, -3.1336e-03, -3.5503e-03, -1.4791e-03,\n",
      "         1.7565e-03, -5.6657e-04, -1.1071e-03, -6.4914e-04,  1.4017e-03,\n",
      "        -1.3294e-03, -2.3772e-03,  1.1744e-03, -4.3565e-04, -1.8504e-04,\n",
      "        -1.9906e-03,  1.9848e-03, -2.4520e-05, -2.3350e-03,  2.9460e-04,\n",
      "        -3.5039e-03, -7.6211e-04,  1.7275e-04, -1.1632e-03,  2.2528e-03,\n",
      "         7.3431e-04, -6.5026e-04,  8.7160e-04,  4.2560e-04, -1.6133e-03,\n",
      "         9.3362e-05,  1.0851e-03, -2.8602e-04,  3.2236e-04, -2.2076e-03,\n",
      "         5.8956e-05, -2.8234e-03, -1.0016e-03, -2.1033e-03, -1.0425e-03,\n",
      "         9.9533e-04,  9.6596e-04, -3.1476e-03,  4.6367e-04,  2.7298e-03,\n",
      "        -2.0229e-03, -2.1049e-03,  6.6876e-04, -1.8535e-03, -5.6679e-05,\n",
      "        -7.7675e-05,  2.2785e-03, -1.5270e-05, -1.9210e-04,  3.0042e-03,\n",
      "        -1.2901e-03, -3.9441e-03, -1.1094e-03, -4.1713e-04, -2.6353e-03,\n",
      "         5.4411e-04, -1.2734e-03, -5.4408e-04,  3.7740e-03,  2.9732e-04,\n",
      "        -1.7132e-03,  1.2852e-03, -2.4745e-03,  2.6074e-03,  4.3728e-04,\n",
      "         1.5723e-03, -1.6024e-03,  7.8164e-04,  3.7156e-04,  4.4426e-04,\n",
      "        -1.0038e-03, -6.1801e-05, -7.1969e-04,  9.4024e-04,  3.9863e-04,\n",
      "        -1.8850e-03, -1.4031e-03,  1.4671e-03,  1.6348e-03,  9.6288e-04,\n",
      "        -6.4040e-04,  8.2771e-04,  8.7670e-04,  1.7046e-03,  8.4886e-04,\n",
      "         8.3920e-04, -1.7850e-03,  6.0865e-04, -8.3508e-04, -5.1231e-04,\n",
      "         1.8322e-03,  5.1957e-04, -2.2339e-04, -5.3185e-04, -3.6484e-04,\n",
      "         1.8443e-03,  7.6294e-05, -1.5738e-03, -2.2887e-03, -2.5483e-03,\n",
      "         3.7205e-04, -1.3101e-05,  1.2532e-04, -9.4033e-04,  1.4380e-04,\n",
      "         2.6685e-04, -1.3443e-03,  2.0838e-04,  2.2581e-03, -1.1899e-03,\n",
      "        -8.0079e-05, -1.0763e-03, -1.5600e-03, -5.0049e-04,  8.8594e-04,\n",
      "        -3.1583e-04,  8.1541e-05,  1.2892e-03, -6.1503e-04, -3.6053e-04,\n",
      "         1.5910e-04, -1.0726e-03, -2.4003e-04, -4.5953e-04, -1.4625e-04,\n",
      "         3.2735e-03,  9.0615e-04, -1.4808e-03, -1.7693e-04,  9.5798e-04,\n",
      "         5.0175e-04,  6.0280e-04,  8.7080e-04, -6.8727e-04, -7.5045e-04,\n",
      "         3.5841e-04,  6.9670e-04,  8.7551e-05, -1.6280e-04,  2.7649e-04,\n",
      "         9.2588e-04, -1.7057e-04, -2.3857e-03, -6.8680e-04, -7.3026e-05,\n",
      "        -1.5772e-03, -5.6985e-04,  4.6946e-03, -1.6109e-03, -7.7609e-04,\n",
      "         3.7121e-04, -1.7002e-03,  1.6345e-03,  2.4960e-04, -1.0844e-03,\n",
      "        -2.4227e-04,  5.0043e-03,  3.4291e-03,  1.5081e-03,  1.6735e-03,\n",
      "         1.5978e-03, -1.4241e-04,  2.1998e-03,  4.8301e-04,  3.7010e-04,\n",
      "         7.5053e-04,  2.0427e-03, -1.5967e-03, -8.6788e-04,  2.0850e-03,\n",
      "        -7.0897e-04,  9.9561e-04,  6.2155e-04,  1.3913e-03, -1.4198e-03,\n",
      "        -1.7903e-03,  1.2489e-03, -2.1325e-03,  2.1157e-03,  5.6359e-03,\n",
      "         1.3841e-03,  2.1318e-03,  2.7837e-03,  1.9447e-03,  2.9123e-03,\n",
      "        -1.9955e-03,  7.8568e-04,  8.7946e-04, -4.9158e-04, -1.2502e-03,\n",
      "        -4.0098e-03,  6.1108e-03, -1.6243e-03, -1.7998e-03, -7.0694e-04,\n",
      "        -9.8011e-04,  4.0274e-05,  1.3018e-03,  2.8780e-03,  3.7944e-03,\n",
      "         8.8249e-04,  1.3090e-03,  3.8061e-04,  5.8689e-04])), ('bert.encoder.layer.4.attention.self.value.weight', tensor([[ 0.0241, -0.0031, -0.0094,  ..., -0.0185,  0.0093, -0.0040],\n",
      "        [ 0.0268, -0.0447,  0.0151,  ...,  0.0517, -0.0521,  0.0260],\n",
      "        [ 0.0157, -0.0482, -0.0286,  ..., -0.0066, -0.0428,  0.0295],\n",
      "        ...,\n",
      "        [ 0.0208, -0.0120,  0.0588,  ..., -0.0062,  0.0242,  0.0002],\n",
      "        [-0.0119, -0.0269, -0.0357,  ..., -0.0077, -0.0328, -0.0136],\n",
      "        [ 0.0046, -0.0186,  0.0327,  ..., -0.0140, -0.0047, -0.0388]])), ('bert.encoder.layer.4.attention.self.value.bias', tensor([-1.4747e-02,  8.0301e-03,  2.2125e-02,  2.7552e-02, -7.4261e-03,\n",
      "        -3.8223e-03,  3.3343e-04, -1.0379e-02, -1.5339e-02,  1.8861e-02,\n",
      "         1.9923e-02, -5.8640e-02, -1.5555e-02, -1.3271e-02,  4.7227e-03,\n",
      "        -6.2322e-03,  2.4308e-02,  3.8249e-02, -1.0723e-02,  2.0570e-02,\n",
      "        -3.5386e-05,  7.1190e-03, -1.8256e-02, -4.8424e-03, -1.9573e-02,\n",
      "         3.4391e-02,  4.1048e-02,  8.1511e-03,  7.8653e-03,  2.3841e-02,\n",
      "         3.7901e-02,  1.2236e-02,  3.1641e-02, -1.5174e-02, -1.5731e-02,\n",
      "        -2.0545e-02, -8.0797e-03,  3.6839e-03, -1.0564e-02, -2.7069e-02,\n",
      "         2.2930e-02,  7.4713e-03, -3.1029e-02, -1.9398e-02,  2.0258e-02,\n",
      "        -4.6119e-03, -4.3719e-02,  6.7184e-04, -5.1929e-03, -6.5470e-03,\n",
      "        -2.2289e-02, -1.9389e-02,  2.5757e-03,  7.3961e-03,  1.6980e-02,\n",
      "         5.6761e-04, -1.4019e-02, -1.4070e-02,  1.6074e-02, -8.7018e-03,\n",
      "        -3.7479e-03,  2.0479e-02, -1.9240e-02, -2.0980e-02, -1.1698e-02,\n",
      "         2.5666e-02, -6.6983e-02, -1.6665e-02, -1.3838e-01,  1.8861e-02,\n",
      "         6.4364e-02, -1.9617e-02, -3.1711e-02, -1.0057e-02, -5.2637e-03,\n",
      "         3.7309e-03,  7.4747e-02, -2.3871e-02,  1.0618e-02, -2.5807e-02,\n",
      "        -1.1890e-01, -6.7743e-02,  7.4459e-02, -5.7745e-02,  1.1727e-01,\n",
      "         9.6919e-03, -9.9545e-02, -4.9518e-02, -7.6977e-02,  3.8803e-02,\n",
      "         8.5890e-02,  3.2180e-02, -6.3481e-02,  7.5920e-03,  9.6739e-02,\n",
      "        -1.0021e-01,  1.6366e-04,  2.0693e-02, -1.8751e-02, -6.0330e-04,\n",
      "        -1.5259e-02,  4.9445e-03,  2.0244e-02, -7.9762e-03,  1.4506e-02,\n",
      "        -2.1430e-02, -9.4977e-03, -1.5307e-02, -8.3026e-03,  1.7546e-02,\n",
      "        -3.3943e-03, -1.6540e-02, -4.5850e-03, -3.2002e-02, -2.6840e-03,\n",
      "        -1.0776e-02,  1.1747e-02, -2.9080e-02, -1.3184e-03, -1.0890e-02,\n",
      "        -1.3331e-03,  1.2047e-02,  1.4541e-02,  1.4178e-02, -1.1809e-03,\n",
      "        -1.9477e-02, -8.9321e-03,  3.4450e-02, -1.1107e-02, -1.4068e-03,\n",
      "        -2.6779e-03,  6.0002e-03, -2.3572e-03, -4.9397e-03,  5.4116e-03,\n",
      "         1.2635e-02,  1.3651e-02, -2.7158e-02, -1.3826e-02, -3.8110e-03,\n",
      "         4.6658e-03,  1.9416e-03,  1.5730e-03, -8.0855e-03, -1.0977e-02,\n",
      "         5.0127e-03,  3.6827e-04,  5.6273e-04,  1.4883e-02,  6.5677e-03,\n",
      "         1.3833e-02,  9.5576e-03, -2.5279e-02, -7.7123e-04, -6.0948e-05,\n",
      "        -6.5342e-03, -1.9074e-03, -7.2399e-03, -1.1319e-03,  6.2778e-03,\n",
      "        -2.6975e-02, -1.2776e-03, -3.0309e-02,  2.1488e-03, -2.7375e-03,\n",
      "        -9.5060e-03, -2.1332e-02, -1.1927e-03,  6.3742e-03,  2.4582e-02,\n",
      "         1.4358e-02,  1.0151e-02, -2.3721e-02, -1.5705e-02, -8.1293e-03,\n",
      "         2.6084e-02, -1.2955e-02, -2.8714e-02,  1.6311e-02,  7.8881e-03,\n",
      "        -3.9940e-04, -8.2851e-03, -1.3446e-02,  6.6567e-03,  1.4207e-02,\n",
      "         1.2702e-02,  5.9110e-03,  7.6114e-03,  9.5486e-03,  3.8187e-02,\n",
      "         4.8544e-03, -6.7487e-03, -6.8739e-03, -1.6344e-02, -2.4506e-02,\n",
      "        -2.8580e-02, -2.5888e-03, -4.3294e-02, -1.3196e-02, -1.5854e-03,\n",
      "         1.3934e-02, -4.7051e-03, -3.4976e-02, -1.1191e-02, -5.0539e-03,\n",
      "         3.3353e-02, -1.5296e-02,  2.3983e-03, -8.3392e-03, -3.1812e-02,\n",
      "         8.1538e-03,  6.3561e-03,  3.5849e-03,  1.2923e-02, -1.0347e-02,\n",
      "        -2.7620e-02, -1.4001e-02, -2.4008e-02, -4.8525e-03, -2.1271e-02,\n",
      "        -1.4826e-02,  1.2272e-02, -1.0540e-02, -1.7456e-02, -7.8369e-03,\n",
      "        -9.6160e-04, -4.8790e-04,  2.7676e-02, -1.2323e-02, -1.8132e-02,\n",
      "        -2.4746e-02,  3.4593e-02, -6.7167e-03, -2.0823e-03,  1.8970e-02,\n",
      "        -1.3802e-02, -1.7251e-02,  1.2644e-02,  1.0721e-02,  3.6402e-03,\n",
      "         1.7762e-02,  2.1914e-02,  7.8763e-03, -1.9061e-02, -1.8420e-02,\n",
      "        -1.4150e-02,  1.3371e-02, -1.0795e-03,  2.5646e-02, -2.3175e-02,\n",
      "         2.9953e-06,  3.0919e-03,  1.1882e-02,  1.6679e-02, -8.7108e-03,\n",
      "         1.2180e-02,  2.1292e-02, -4.4371e-02,  3.0415e-02, -5.7430e-03,\n",
      "         2.6856e-02,  2.6405e-03,  9.1889e-04, -3.4612e-02, -2.7549e-02,\n",
      "        -1.5038e-02,  3.2081e-02,  3.4948e-02, -1.0997e-02,  3.0718e-03,\n",
      "        -2.9645e-02, -1.1096e-02,  3.5954e-03, -1.3193e-02,  6.7647e-03,\n",
      "         1.1795e-02,  1.1144e-02, -7.9865e-03,  1.4255e-02,  2.8918e-02,\n",
      "         3.7005e-02,  1.3883e-02,  3.1100e-02, -1.6183e-02,  1.4706e-02,\n",
      "        -2.0742e-02,  8.0462e-03,  1.3717e-02,  8.6716e-03,  1.7882e-02,\n",
      "        -2.3850e-03,  1.5839e-02,  5.4857e-03, -1.2506e-02, -1.1707e-02,\n",
      "         9.4343e-03,  1.0343e-02, -1.5608e-02, -7.4038e-04,  8.0737e-03,\n",
      "         2.5322e-02,  8.1714e-03, -1.5135e-02,  1.5930e-02, -1.4792e-02,\n",
      "        -6.3729e-03, -6.4740e-03, -1.7648e-02,  2.2624e-02, -5.8700e-03,\n",
      "         1.5435e-02, -1.5833e-03, -3.4476e-03, -2.0601e-03, -1.0052e-03,\n",
      "        -1.6819e-03,  2.1769e-02,  1.2614e-02,  1.9899e-03,  1.8929e-02,\n",
      "        -8.5147e-02,  1.5813e-02, -1.5564e-02,  2.9195e-02, -2.7869e-02,\n",
      "         2.3505e-02,  6.8040e-02, -1.7945e-02, -2.3189e-02,  9.7383e-03,\n",
      "         2.8831e-02, -1.7791e-02, -6.0974e-02, -1.5484e-02, -9.1812e-03,\n",
      "        -1.9617e-02,  3.6750e-02, -1.9438e-03, -1.0636e-02,  3.5327e-03,\n",
      "         3.5047e-02, -1.0278e-02,  3.5408e-02,  5.3865e-03,  4.0022e-02,\n",
      "         1.9354e-02,  5.9103e-03, -7.7096e-03,  2.6439e-02, -4.1608e-02,\n",
      "         3.9228e-02, -3.6683e-02,  3.8163e-03, -1.5099e-02, -9.3826e-03,\n",
      "         3.8262e-02, -1.9064e-02, -3.7401e-02, -3.8823e-02,  2.1826e-02,\n",
      "        -3.4592e-03,  4.7983e-02, -9.3169e-03, -9.7342e-03,  7.7368e-03,\n",
      "        -3.2218e-02,  6.3254e-04, -2.9806e-02,  5.9972e-03, -3.0323e-03,\n",
      "        -3.3491e-02, -2.6278e-03, -6.1879e-03,  3.3920e-03,  1.2476e-02,\n",
      "        -3.4262e-03, -3.3415e-03,  1.3386e-02,  2.1095e-02, -6.4803e-03,\n",
      "         1.2437e-03, -1.2049e-02,  1.4192e-02, -9.3611e-03])), ('bert.encoder.layer.4.attention.output.dense.weight', tensor([[-0.0173, -0.0298, -0.0757,  ...,  0.0257,  0.0249,  0.0014],\n",
      "        [ 0.0620, -0.0022,  0.0319,  ...,  0.0265,  0.0015, -0.0085],\n",
      "        [-0.0098,  0.0264, -0.0488,  ...,  0.0229,  0.0064,  0.0354],\n",
      "        ...,\n",
      "        [ 0.0286,  0.0345, -0.0128,  ...,  0.0074,  0.0100, -0.0399],\n",
      "        [-0.0134, -0.0331,  0.0159,  ..., -0.0165,  0.0001,  0.0208],\n",
      "        [-0.0248, -0.0329,  0.0535,  ..., -0.0061,  0.0318,  0.0279]])), ('bert.encoder.layer.4.attention.output.dense.bias', tensor([-5.0580e-02, -4.0130e-02,  7.2692e-02, -2.2278e-02,  7.3094e-02,\n",
      "         2.3124e-02,  2.0815e-02, -6.0052e-02, -1.6979e-01, -4.1421e-02,\n",
      "         4.6243e-02, -3.3356e-02, -5.8299e-02, -2.8632e-03, -3.9414e-03,\n",
      "         8.0732e-03, -3.6403e-02, -5.4369e-02,  5.9253e-03, -5.1792e-03,\n",
      "         5.0812e-02,  1.2398e-02,  2.4525e-02,  1.0977e-01,  5.1381e-02,\n",
      "        -5.1100e-02,  4.6283e-02,  4.6881e-02,  8.7244e-04,  2.3681e-02,\n",
      "        -2.6638e-02, -2.5851e-03, -3.6474e-03,  3.5235e-02, -7.4762e-02,\n",
      "         4.9886e-02,  8.8584e-02, -7.6640e-02, -5.2239e-02,  3.4526e-02,\n",
      "         3.6667e-02, -3.9224e-03,  2.5899e-04, -2.6225e-02,  3.4769e-02,\n",
      "         1.8407e-02,  3.4118e-02,  3.1916e-02, -5.7784e-02, -1.5958e-02,\n",
      "         5.7638e-03,  1.1763e-03,  7.6363e-03,  2.2971e-02, -6.8167e-02,\n",
      "         2.8482e-02, -8.0219e-02,  6.4448e-02, -8.1901e-02, -5.5745e-02,\n",
      "        -3.4452e-02, -3.6350e-02, -7.7011e-02, -2.0010e-02,  3.2535e-02,\n",
      "         3.5342e-03, -2.5399e-02,  2.7346e-02,  2.8690e-02,  5.5651e-02,\n",
      "         4.1220e-02,  7.3615e-02,  4.5982e-03,  3.1537e-02, -3.9084e-02,\n",
      "         8.8419e-03,  3.9479e-02, -4.7402e-02, -1.3419e-03, -5.0304e-02,\n",
      "        -9.2114e-03,  9.9971e-03,  5.1640e-02, -4.6684e-02, -3.7100e-02,\n",
      "        -1.1909e-02,  1.6067e-02,  4.2164e-02, -7.0966e-02,  3.6134e-02,\n",
      "        -7.6504e-03,  4.2994e-02, -9.0634e-03, -7.1306e-02,  4.3676e-03,\n",
      "         5.5442e-03,  2.2872e-02,  1.3985e-02, -8.6750e-03,  2.5011e-02,\n",
      "        -4.9762e-02, -8.4228e-03,  3.8284e-03,  3.7455e-02,  3.7102e-02,\n",
      "         1.4943e-02,  4.9690e-02, -9.6590e-03, -2.7128e-02,  4.1890e-02,\n",
      "        -2.2135e-02,  2.8301e-02, -1.8041e-02,  3.9716e-02,  4.5780e-03,\n",
      "         6.3871e-02, -4.0966e-03, -1.0726e-02,  1.5319e-02, -6.3715e-02,\n",
      "         2.4166e-02,  1.3419e-02,  6.5340e-03,  1.5198e-02,  2.3032e-02,\n",
      "        -1.6065e-02,  2.7430e-02, -7.8677e-02,  2.0668e-02,  3.8824e-02,\n",
      "        -1.6414e-02,  2.5076e-02,  2.6970e-02,  1.1499e-02,  3.6468e-02,\n",
      "        -4.6122e-02, -5.2241e-03, -1.0595e-03,  6.8706e-03,  9.3070e-03,\n",
      "         7.8516e-03,  1.1043e-02, -3.0132e-02,  1.8143e-02, -1.6402e-02,\n",
      "        -4.0385e-02,  1.4509e-02,  1.4436e-03, -8.4844e-03, -1.0250e-02,\n",
      "         2.3048e-03,  2.9654e-02,  4.6420e-02,  1.4687e-02, -3.4423e-02,\n",
      "        -2.2145e-02,  1.8726e-02, -1.1702e-02, -3.7352e-02,  2.3485e-02,\n",
      "        -1.6699e-02, -3.7405e-03,  3.5446e-02, -9.7040e-03, -5.0582e-03,\n",
      "         5.1000e-02, -1.8171e-02,  4.2505e-02,  1.5746e-02,  8.2304e-03,\n",
      "         2.4508e-03, -5.9044e-02,  1.6648e-02,  2.5986e-02, -7.3984e-03,\n",
      "        -1.4846e-03,  8.6323e-04,  7.0716e-03, -2.1231e-02,  9.4185e-02,\n",
      "        -1.9399e-03,  1.6165e-02, -4.6101e-02,  5.0075e-02, -4.5300e-02,\n",
      "        -1.5864e-02, -2.3273e-02, -3.9577e-02,  7.6372e-03,  1.8299e-02,\n",
      "        -4.6745e-02,  4.3636e-02, -5.7113e-02, -2.9574e-02, -5.3381e-02,\n",
      "        -2.7186e-03,  2.8613e-02,  4.1937e-02, -9.1495e-02,  1.6641e-02,\n",
      "        -6.4123e-02, -1.9915e-02,  6.9926e-02,  4.1876e-02, -4.3519e-02,\n",
      "         6.0532e-02,  1.2641e-02,  4.0327e-02, -1.9115e-03, -1.6893e-02,\n",
      "         3.2195e-02, -1.5956e-02,  1.6888e-02,  4.0536e-02, -4.9594e-02,\n",
      "         8.2228e-02, -2.7713e-02, -1.9934e-02, -1.9495e-02,  9.6171e-03,\n",
      "        -2.7765e-02, -4.7249e-03, -2.5494e-01,  1.8703e-02, -6.6067e-02,\n",
      "        -6.9186e-03,  5.8162e-02, -1.3528e-02, -3.3324e-02,  4.8313e-02,\n",
      "         3.0398e-02, -1.2286e-03,  1.2004e-02, -1.0554e-04,  5.1483e-02,\n",
      "        -1.5559e-03, -3.0083e-02, -7.3712e-03, -9.2256e-03,  5.7639e-03,\n",
      "         2.2736e-02,  6.9361e-04,  1.4363e-02,  3.0599e-02,  3.9354e-02,\n",
      "         7.3364e-02, -7.1618e-03, -6.9670e-03, -5.4380e-02,  1.3447e-02,\n",
      "         5.1860e-03,  1.9936e-02,  2.0255e-02, -2.1393e-02, -2.3396e-02,\n",
      "         6.3641e-02, -1.8462e-02, -3.2664e-02, -8.4493e-02, -3.7100e-02,\n",
      "         3.4095e-02, -2.6551e-02, -2.3804e-02,  1.9541e-02,  1.0800e-02,\n",
      "         4.3910e-02,  3.2656e-03, -6.1576e-03,  4.1370e-02, -1.5063e-02,\n",
      "        -2.0139e-02,  6.4105e-02, -2.3969e-02,  4.4333e-02,  1.2562e-02,\n",
      "         2.3861e-01, -5.6955e-03,  2.1381e-02, -7.7614e-03,  2.6735e-02,\n",
      "        -2.6596e-02, -2.5176e-02, -4.1992e-02, -4.7854e-02,  1.3484e-02,\n",
      "         1.0281e-03, -5.5663e-02,  3.8051e-02, -3.6073e-02,  4.5235e-03,\n",
      "        -2.1446e-02,  1.7817e-02,  3.3542e-03,  1.9211e-02, -1.3480e-02,\n",
      "         1.2677e-02, -2.1753e-02, -4.5882e-02, -1.0929e-02,  6.6831e-02,\n",
      "        -6.3653e-02, -2.6919e-02,  7.0212e-02, -2.2099e-02, -1.7036e-02,\n",
      "         2.7945e-02, -1.3768e-02,  1.1021e-02,  2.6917e-02,  2.7268e-02,\n",
      "         9.1059e-03, -3.1056e-02, -1.4973e-03, -1.5657e-02,  1.7019e-03,\n",
      "         7.3370e-02, -3.6019e-02,  3.6232e-02, -8.5695e-04,  3.1512e-02,\n",
      "        -3.5849e-03, -5.7262e-03,  4.7642e-02,  1.8774e-02, -6.5121e-03,\n",
      "         2.3706e-02,  2.7430e-03,  1.9623e-02,  2.7449e-02, -1.4870e-03,\n",
      "        -1.2943e-02,  6.6607e-03, -4.6682e-02,  6.6200e-02, -2.7597e-02,\n",
      "         6.4246e-02, -8.1079e-02,  3.4154e-02, -4.0408e-02,  2.8103e-02,\n",
      "         6.7828e-04,  2.3463e-02,  3.3718e-02, -4.3112e-03, -1.8122e-02,\n",
      "         1.1731e-02, -3.3344e-02,  1.1998e-02, -3.5715e-02, -4.8867e-03,\n",
      "        -8.8168e-02, -1.6985e-02,  2.6295e-02,  1.1469e-02, -5.0239e-02,\n",
      "        -1.6687e-02, -5.7712e-02,  1.5593e-02, -3.7599e-02,  3.0934e-02,\n",
      "        -8.0756e-03,  1.2374e-02,  2.1214e-02,  2.9672e-02,  3.6302e-02,\n",
      "        -5.3380e-02,  7.3492e-03, -4.7043e-02, -5.5232e-02, -1.0906e-02,\n",
      "         9.7504e-03,  4.7019e-03, -2.9532e-02,  2.8568e-02,  3.3520e-02,\n",
      "         3.8689e-02,  4.8893e-02, -4.3476e-02, -7.1955e-02,  3.2334e-02,\n",
      "         3.2542e-02, -6.3788e-02, -4.1625e-02, -2.7801e-02])), ('bert.encoder.layer.4.attention.output.LayerNorm.weight', tensor([0.8200, 0.8104, 0.7694, 0.7491, 0.9770, 0.8086, 0.7589, 0.8526, 0.8223,\n",
      "        0.7998, 0.8371, 0.8153, 0.7704, 0.7997, 0.8301, 0.7662, 0.8226, 0.7806,\n",
      "        0.7912, 0.7970, 0.7711, 0.7660, 0.7923, 0.7874, 0.7511, 0.7632, 0.7574,\n",
      "        0.8015, 0.7710, 0.7657, 0.8667, 0.8558, 0.8676, 0.7790, 0.8464, 0.8425,\n",
      "        0.8401, 0.8432, 0.8266, 0.8029, 0.7816, 0.7636, 0.8207, 0.8280, 0.7982,\n",
      "        0.8125, 0.7676, 0.8858, 0.7942, 0.8476, 0.7993, 0.7846, 0.7590, 0.8171,\n",
      "        0.8335, 0.8190, 0.8546, 0.9122, 0.8659, 0.9022, 0.8570, 0.8935, 0.8504,\n",
      "        0.8669, 0.8715, 0.9629, 0.9524, 0.9250, 0.9068, 0.9180, 0.8741, 0.9669,\n",
      "        0.8835, 0.9257, 0.9101, 0.9128, 0.9265, 0.8816, 0.8876, 0.9093, 0.9953,\n",
      "        0.8535, 0.9150, 0.9072, 0.8951, 0.8878, 0.9521, 0.9871, 0.8855, 0.9337,\n",
      "        0.9447, 0.9048, 0.9698, 0.9279, 0.8853, 0.9270, 0.9598, 0.9704, 0.9407,\n",
      "        0.9920, 0.9685, 0.8490, 0.9356, 0.9790, 0.9622, 0.8440, 0.9689, 0.8996,\n",
      "        0.9539, 0.9539, 0.9088, 0.9498, 0.9873, 0.9039, 0.9791, 0.9333, 0.9284,\n",
      "        0.9871, 0.9586, 0.9494, 1.2100, 0.9879, 0.9921, 0.9546, 0.9634, 0.9243,\n",
      "        0.9607, 0.9469, 0.9016, 0.9632, 0.8968, 0.9561, 0.8942, 0.9726, 0.9720,\n",
      "        0.9983, 0.9472, 0.9474, 0.9243, 0.9543, 0.9481, 1.0033, 0.9800, 0.9586,\n",
      "        0.8697, 0.9512, 0.9428, 0.9703, 0.9657, 0.9864, 0.9288, 0.9626, 0.9336,\n",
      "        0.9603, 0.8455, 0.9376, 0.8799, 0.9376, 0.9321, 0.9584, 0.9738, 0.9614,\n",
      "        0.9176, 0.9743, 0.9707, 0.8773, 0.9766, 0.8770, 0.8651, 0.9485, 0.9558,\n",
      "        0.9594, 0.9230, 0.9661, 0.9790, 0.9870, 0.9507, 0.9285, 0.9665, 0.9919,\n",
      "        0.9841, 0.8611, 0.9383, 0.9455, 0.9341, 0.9453, 0.9836, 1.1716, 1.0005,\n",
      "        0.9184, 0.9456, 0.9131, 0.9410, 0.9253, 0.9955, 0.9419, 0.9859, 0.9539,\n",
      "        0.9690, 0.9271, 0.9578, 0.9455, 0.9410, 0.9668, 0.9624, 0.9144, 0.9509,\n",
      "        0.9497, 0.9984, 0.9351, 0.9543, 0.9465, 0.9365, 0.9729, 0.9586, 0.9556,\n",
      "        0.9233, 0.9603, 0.9045, 0.9799, 0.9698, 0.9612, 1.2966, 0.9092, 0.9958,\n",
      "        0.9687, 0.9246, 0.9491, 0.9812, 0.9232, 0.9512, 0.9612, 0.9170, 0.9470,\n",
      "        0.9586, 0.9963, 0.8748, 0.9753, 0.9170, 0.9263, 0.9042, 0.9767, 0.9384,\n",
      "        0.9306, 0.9608, 0.9694, 0.9449, 0.9264, 0.9882, 0.9540, 0.9637, 1.0023,\n",
      "        1.0074, 1.0067, 0.9542, 0.9940, 0.9536, 0.9632, 1.0074, 0.9588, 0.9769,\n",
      "        0.9986, 0.9602, 0.9384, 0.9289, 0.9798, 0.9400, 0.9614, 0.8740, 0.9248,\n",
      "        0.9681, 1.0207, 0.9721, 0.9462, 0.9035, 1.4330, 1.0077, 0.9565, 1.0054,\n",
      "        0.9780, 0.9886, 0.9696, 0.9530, 0.8945, 0.9003, 0.9726, 0.9661, 0.8728,\n",
      "        0.8909, 0.9484, 0.9657, 0.9549, 0.9732, 0.9338, 0.9163, 0.9621, 0.9559,\n",
      "        0.9605, 0.9856, 1.0063, 1.0805, 0.9319, 0.9533, 0.9477, 0.9618, 0.9248,\n",
      "        0.9745, 0.9457, 0.9592, 0.9287, 0.9657, 0.9367, 0.9660, 0.9702, 0.9774,\n",
      "        0.9582, 0.9737, 0.9824, 0.9819, 0.9654, 0.9607, 0.9537, 0.9775, 0.9705,\n",
      "        0.9652, 0.9635, 0.9958, 0.9203, 0.9410, 0.9434, 0.9305, 0.9235, 0.9664,\n",
      "        0.9582, 0.9408, 1.0552, 1.0048, 0.9259, 0.9769, 0.9885, 0.9679, 0.9732,\n",
      "        0.9573, 0.9181, 0.9693, 0.9859, 0.9579, 0.9812, 1.1076, 1.0229, 1.0221,\n",
      "        0.9878, 0.8942, 0.9130, 0.9803, 0.9864, 1.0256, 0.9982, 0.9612, 0.9573,\n",
      "        0.9517, 0.9554, 0.9350, 0.8561, 1.1097, 1.0327, 0.9565, 0.9259, 0.9829,\n",
      "        0.8436, 0.9633, 0.9729, 0.9230, 0.9553, 0.9635, 0.9437, 0.9179, 0.9039,\n",
      "        1.0013, 0.9164, 0.9497, 1.0114, 0.9525, 0.9473])), ('bert.encoder.layer.4.attention.output.LayerNorm.bias', tensor([ 4.1961e-02,  2.9045e-02,  6.5173e-02, -1.0612e-02,  1.5644e-01,\n",
      "        -1.2323e-01,  6.2860e-02, -2.7159e-02, -2.3386e-01, -1.7331e-02,\n",
      "         2.0142e-01, -3.1623e-03, -1.3710e-01, -1.8476e-01, -2.2832e-01,\n",
      "         1.6235e-02,  3.4283e-02, -6.0356e-02,  3.3418e-02, -5.5254e-02,\n",
      "         4.2064e-02,  1.5833e-01, -4.2582e-02,  1.8496e-01,  1.4125e-01,\n",
      "        -4.0857e-02,  1.9867e-02,  7.8060e-02, -3.2759e-02, -1.5288e-02,\n",
      "        -9.2947e-02, -4.1995e-03, -9.3936e-02, -1.4076e-01, -2.5328e-01,\n",
      "         1.4791e-01,  1.3141e-01, -3.9056e-02, -4.8948e-02, -2.4308e-03,\n",
      "        -4.0116e-02, -8.6209e-02, -8.3905e-02, -1.6058e-01, -9.1661e-03,\n",
      "         1.2322e-01,  3.0673e-02,  1.1701e-01, -6.7941e-02,  7.3612e-02,\n",
      "         1.7477e-01, -6.3112e-02,  7.0511e-02,  1.2967e-01, -1.6072e-01,\n",
      "         1.2839e-01, -1.1322e-01,  1.6226e-01, -1.9622e-01, -1.9668e-01,\n",
      "         1.3749e-02,  1.2724e-01, -1.3443e-01,  7.3786e-02,  3.0432e-02,\n",
      "         6.0675e-02,  4.2162e-02,  2.4995e-02,  1.0150e-02,  4.4431e-02,\n",
      "         7.9842e-02,  1.0489e-01,  1.0803e-01,  1.5090e-01, -9.7483e-02,\n",
      "         2.1605e-02,  7.5489e-02,  4.4853e-02,  1.1433e-02, -9.2900e-02,\n",
      "         1.4333e-02,  2.4241e-02,  2.6492e-01, -8.9474e-02, -5.6141e-02,\n",
      "        -7.6302e-02, -8.6552e-02,  1.7343e-01, -5.5599e-02,  2.0213e-01,\n",
      "         8.0075e-02, -7.6133e-02, -1.2551e-01,  1.3349e-01,  7.1877e-02,\n",
      "         1.1826e-01,  2.4556e-02,  1.0096e-01, -1.2296e-01,  6.3927e-02,\n",
      "        -1.0047e-01,  2.1773e-02,  1.5891e-02,  1.7501e-01, -2.9001e-02,\n",
      "        -7.0237e-02, -1.3568e-02,  8.1050e-02, -5.4844e-02,  2.0327e-01,\n",
      "         8.1387e-02,  1.0922e-01, -2.0034e-01,  4.5361e-02,  7.8953e-02,\n",
      "         1.4542e-01,  4.3081e-03, -7.6468e-02,  1.4736e-01, -1.2233e-01,\n",
      "         1.2437e-01,  5.8566e-02,  1.1693e-01,  7.0052e-02,  9.5528e-02,\n",
      "        -6.0456e-02,  2.6946e-02, -1.8636e-01, -5.5135e-03,  1.1960e-01,\n",
      "        -7.7181e-02,  1.9172e-01, -1.3146e-02, -5.7476e-02,  7.8005e-02,\n",
      "         9.2574e-02,  1.1545e-01, -5.0488e-02, -5.1107e-02, -8.4105e-02,\n",
      "        -5.5861e-02,  6.1043e-02, -7.3011e-02, -8.0745e-02, -3.4163e-02,\n",
      "         3.5576e-02, -2.7942e-02,  5.6526e-02, -1.3726e-01,  6.8747e-02,\n",
      "         1.0812e-01,  1.6525e-01, -6.8759e-03, -1.9813e-02, -4.0224e-02,\n",
      "        -8.7648e-02,  3.5332e-02,  1.0556e-01, -1.3806e-01,  1.3739e-01,\n",
      "        -7.0515e-02, -2.8352e-02,  4.5667e-02,  8.3195e-02, -5.3476e-02,\n",
      "         6.6025e-02, -1.8231e-02,  7.1859e-02, -1.6716e-02,  1.1091e-01,\n",
      "        -2.5943e-03, -2.3986e-01, -3.1027e-02,  1.4853e-01,  4.5191e-02,\n",
      "         6.7241e-02, -4.4363e-02,  1.5226e-01, -1.1287e-01,  2.7043e-01,\n",
      "        -5.8370e-02,  6.6399e-02, -4.5978e-02,  1.1764e-01, -1.2134e-01,\n",
      "        -4.0355e-02,  2.1523e-02,  8.0414e-02,  8.9191e-03,  9.4299e-02,\n",
      "        -7.4102e-02,  9.5667e-02, -8.9529e-02, -1.2732e-01, -2.1785e-01,\n",
      "         1.5729e-01,  1.4274e-01,  8.8744e-02, -1.6245e-01, -3.0343e-02,\n",
      "        -6.2225e-02, -3.1014e-03,  1.2565e-01,  9.8480e-02, -1.1469e-01,\n",
      "         1.0514e-01,  3.9294e-02,  1.4410e-01, -4.0147e-02,  1.4475e-01,\n",
      "        -7.5801e-02,  1.8045e-03, -5.2153e-02,  1.1473e-01, -2.0877e-02,\n",
      "         1.6550e-01, -9.8902e-02, -1.2383e-01, -1.4156e-01,  2.0319e-02,\n",
      "        -1.9423e-01, -5.3554e-02, -7.1328e-01, -1.0083e-01, -1.7032e-01,\n",
      "        -4.0069e-02,  1.3388e-01,  1.4166e-01, -1.2891e-03,  6.4898e-02,\n",
      "         6.7574e-02, -5.7703e-02, -1.1142e-01, -4.2926e-02,  1.2174e-01,\n",
      "        -5.3878e-03,  2.4311e-02,  2.1926e-02,  2.0481e-02,  2.1714e-02,\n",
      "        -4.7903e-02,  1.3133e-01,  7.3981e-02,  7.8327e-02,  5.0442e-02,\n",
      "         1.8711e-01, -9.8773e-02,  7.3659e-02, -1.4502e-01,  1.3714e-02,\n",
      "         6.9439e-03,  2.1363e-01, -4.9001e-02, -2.4279e-02, -2.2176e-02,\n",
      "         1.7932e-01,  9.0694e-02, -1.9252e-02, -3.2987e-01, -1.0522e-01,\n",
      "         6.0163e-02, -1.7154e-01,  4.1985e-03,  4.8649e-02,  8.9913e-03,\n",
      "         1.6222e-01,  1.4570e-03, -1.8796e-02,  2.2326e-02, -1.3023e-02,\n",
      "        -1.5467e-01,  2.5215e-01, -9.5876e-02,  1.2538e-01,  8.3882e-02,\n",
      "         6.9221e-01, -1.1033e-01,  1.7848e-01, -1.6214e-01,  1.9114e-01,\n",
      "        -9.6053e-02, -9.7136e-02, -5.5771e-04, -4.6427e-02, -3.0967e-02,\n",
      "         8.1532e-02, -1.9288e-01,  2.4135e-01, -1.3004e-01, -9.8044e-02,\n",
      "        -1.3047e-01,  1.2113e-01, -1.3983e-01, -6.6584e-02, -1.8843e-02,\n",
      "         7.7609e-02, -1.1595e-02,  1.3653e-01, -1.3196e-01,  2.5257e-01,\n",
      "        -2.0648e-01, -2.0937e-01,  8.5538e-02,  3.4942e-02, -5.0789e-02,\n",
      "         7.2339e-02, -1.1817e-01,  9.8463e-02, -2.3246e-02,  8.0025e-02,\n",
      "        -8.6604e-02,  1.3794e-01, -4.4238e-02, -1.0995e-02, -1.3740e-01,\n",
      "         1.5323e-01, -1.3529e-01,  1.3730e-01,  9.2352e-02, -1.6077e-02,\n",
      "         5.7206e-02,  1.9459e-02,  6.0170e-04,  1.0307e-01,  1.6515e-02,\n",
      "         8.2621e-02, -1.7561e-01, -6.3221e-02,  3.4046e-02,  1.2154e-01,\n",
      "        -9.5609e-02, -4.7964e-02, -1.2415e-01,  2.3247e-01, -3.8562e-02,\n",
      "         2.6271e-01, -1.8107e-01,  7.0795e-02, -3.0166e-02,  1.5205e-01,\n",
      "         2.7940e-02,  1.3606e-01,  5.9238e-02,  1.0078e-01, -1.6510e-01,\n",
      "         8.8830e-03, -2.7914e-02, -1.7723e-01, -3.3779e-01,  2.2310e-01,\n",
      "        -2.5292e-01,  1.6894e-01, -1.3395e-01, -5.1858e-02, -1.6236e-01,\n",
      "         1.5374e-01, -2.1909e-01,  1.2181e-01, -1.0982e-01,  2.0008e-01,\n",
      "        -1.7234e-01,  6.7705e-02,  1.0258e-01,  1.2284e-01,  4.5651e-02,\n",
      "        -2.0294e-01,  7.3353e-02, -2.4914e-02, -2.3661e-01,  3.9693e-03,\n",
      "        -1.5932e-01, -5.1021e-02, -4.5670e-03, -3.5937e-02,  1.2129e-01,\n",
      "         5.2584e-02,  6.0483e-02, -1.5938e-01, -2.0375e-01,  2.1856e-01,\n",
      "        -1.4900e-02, -1.9883e-01, -1.8389e-01,  3.2183e-02])), ('bert.encoder.layer.4.intermediate.dense.weight', tensor([[ 0.0509,  0.0269,  0.0439,  ...,  0.0211,  0.1031,  0.0050],\n",
      "        [-0.0248,  0.0940, -0.0132,  ...,  0.0944,  0.0757, -0.0491],\n",
      "        [ 0.1124,  0.0123, -0.1371,  ...,  0.0398,  0.0197, -0.0179],\n",
      "        ...,\n",
      "        [ 0.0081, -0.1050,  0.0104,  ..., -0.0153, -0.0262, -0.0159],\n",
      "        [-0.0059,  0.0489, -0.0913,  ...,  0.0087,  0.0123,  0.0168],\n",
      "        [ 0.0518, -0.0633, -0.1221,  ..., -0.0218,  0.0172, -0.0751]])), ('bert.encoder.layer.4.intermediate.dense.bias', tensor([-0.0754, -0.0806, -0.0610,  ..., -0.0852, -0.0882, -0.0300])), ('bert.encoder.layer.4.output.dense.weight', tensor([[ 0.0023, -0.0760, -0.0345,  ...,  0.0085, -0.0244,  0.0332],\n",
      "        [-0.0320,  0.0974, -0.0783,  ..., -0.0142, -0.0700, -0.0137],\n",
      "        [-0.0401,  0.0286,  0.1524,  ..., -0.0291,  0.0155, -0.0372],\n",
      "        ...,\n",
      "        [-0.0307, -0.0279,  0.0599,  ..., -0.0264, -0.0066, -0.0167],\n",
      "        [-0.0083, -0.1137, -0.0089,  ...,  0.0341, -0.0035,  0.0426],\n",
      "        [ 0.1293,  0.0363,  0.0238,  ...,  0.0351, -0.0117, -0.0952]])), ('bert.encoder.layer.4.output.dense.bias', tensor([ 4.2828e-02,  4.1640e-02, -9.4399e-03,  1.3000e-02,  3.3870e-01,\n",
      "        -2.7548e-02,  4.4711e-03,  4.0044e-02, -7.0390e-02,  2.9641e-02,\n",
      "         3.3757e-02, -2.9344e-02, -1.5500e-02, -4.2599e-02, -2.5766e-02,\n",
      "        -1.9562e-02, -3.6319e-02, -3.7561e-02,  4.2954e-02,  3.8932e-03,\n",
      "         4.9184e-03,  8.1180e-02,  3.0855e-02,  2.9820e-02,  6.0700e-02,\n",
      "        -1.5022e-02, -4.8101e-02,  6.4094e-02,  5.0997e-02, -4.7203e-02,\n",
      "        -2.0872e-02,  5.8085e-02, -2.8690e-02, -5.7328e-02, -6.3319e-02,\n",
      "         4.2542e-02,  4.0309e-02, -1.1857e-02,  3.3369e-02,  1.3510e-02,\n",
      "        -3.3718e-03, -5.2361e-02, -3.9741e-02, -5.0966e-02, -3.2463e-02,\n",
      "         3.2472e-02,  4.8391e-02, -1.3730e-02,  1.1915e-02, -4.1593e-02,\n",
      "         9.4286e-02, -1.0261e-01,  3.5892e-02,  5.1189e-02, -1.4722e-02,\n",
      "         6.1641e-02, -3.5968e-02,  8.3554e-02, -4.0710e-02, -4.6694e-02,\n",
      "        -2.1827e-02,  6.6775e-02, -2.2487e-03,  4.6780e-02, -4.3513e-03,\n",
      "         4.3190e-02, -3.1136e-02, -1.4464e-02,  5.8574e-03, -8.5111e-03,\n",
      "         3.7806e-02,  3.8549e-02,  3.8866e-02,  1.9850e-02,  3.2859e-02,\n",
      "         1.2127e-02,  3.3967e-02, -2.5843e-02,  3.6454e-02, -1.4840e-02,\n",
      "        -3.4717e-02, -1.2014e-02,  3.7783e-02, -5.5247e-02, -2.0550e-02,\n",
      "         3.1935e-03, -2.6143e-03, -2.8897e-03, -3.8222e-02,  3.0444e-02,\n",
      "         8.8477e-03, -4.7078e-02, -5.0151e-02,  8.3517e-03,  4.8041e-02,\n",
      "        -9.0790e-03,  3.0854e-02, -7.4392e-03, -1.2810e-02, -1.2030e-02,\n",
      "        -8.3002e-03, -1.6601e-02,  7.1530e-03,  4.3931e-02, -2.8476e-02,\n",
      "        -1.4911e-02,  2.1337e-02, -2.3236e-02, -6.3219e-03,  1.4721e-02,\n",
      "         3.3474e-02, -6.3899e-03, -4.9793e-02,  6.5398e-03, -1.3599e-02,\n",
      "         5.0002e-02,  5.8206e-03, -2.6199e-02,  3.9628e-02, -4.7831e-02,\n",
      "        -6.0979e-02, -7.3938e-03,  2.7536e-02,  8.8301e-03,  3.3661e-02,\n",
      "        -1.0073e-02,  1.7198e-02, -2.7893e-02,  9.4320e-03,  3.3832e-02,\n",
      "        -5.6086e-03,  4.9428e-02, -3.0020e-02, -2.6539e-02,  1.9287e-02,\n",
      "         4.4753e-02,  1.5457e-02, -3.8738e-02, -1.1957e-02, -1.9183e-02,\n",
      "         5.2889e-03, -1.2506e-02,  3.0652e-03, -2.0476e-02,  1.4300e-02,\n",
      "         1.2222e-02, -1.4549e-02, -1.4053e-03, -3.6864e-02,  2.4976e-02,\n",
      "         2.6690e-02,  2.1902e-03, -1.3426e-02, -2.3372e-02, -1.8775e-02,\n",
      "        -2.1006e-02,  1.8485e-02,  5.0503e-03, -4.2448e-02, -2.5858e-02,\n",
      "        -1.6337e-03, -7.9668e-03,  2.4770e-02,  1.0298e-02, -1.6714e-02,\n",
      "         2.3337e-02,  5.4755e-02,  1.2153e-02,  2.3176e-03,  3.1183e-02,\n",
      "        -6.2934e-03,  2.5665e-03,  1.4817e-02,  9.5974e-04,  6.1823e-03,\n",
      "         5.8221e-03, -2.3559e-02,  5.4211e-02, -4.1857e-02,  1.2093e-03,\n",
      "         1.1016e-02,  9.7562e-03, -3.3921e-03,  1.5595e-02,  3.4622e-02,\n",
      "        -2.5396e-02, -2.3996e-03,  5.7491e-03,  4.5363e-04, -1.9025e-02,\n",
      "        -1.2611e-02,  2.5755e-02, -6.4917e-02, -5.4736e-02, -6.8256e-02,\n",
      "        -2.2142e-02,  2.8576e-02,  2.0616e-02, -5.8795e-02, -1.1570e-02,\n",
      "         1.2554e-02,  1.4614e-02,  8.1673e-04, -2.9100e-03, -4.7407e-02,\n",
      "         1.0457e-02, -2.6998e-02,  5.4274e-02, -2.2680e-03,  4.9477e-02,\n",
      "         1.4735e-02, -6.9379e-03,  1.5979e-02,  4.0992e-02,  1.8217e-03,\n",
      "         1.6022e-02, -2.2045e-02, -2.1808e-02,  1.5614e-02,  9.0632e-03,\n",
      "        -7.4418e-02, -8.6354e-03, -1.0716e-01, -1.4357e-02,  5.2110e-03,\n",
      "        -1.4063e-02,  7.0683e-02, -3.7959e-03,  3.4894e-02, -6.0429e-03,\n",
      "         6.6495e-02, -1.3960e-02, -1.4047e-02,  8.2808e-04,  6.3790e-02,\n",
      "         1.4025e-02, -1.8530e-02,  2.3308e-02,  3.8107e-02, -1.4271e-03,\n",
      "         1.3577e-02,  5.2247e-02,  1.4670e-02,  1.2441e-02,  2.0285e-02,\n",
      "        -6.1882e-03,  1.8738e-02, -5.9404e-03, -1.2706e-02,  2.8636e-02,\n",
      "         5.3770e-03,  2.3726e-02, -5.0358e-03,  2.3995e-02,  9.7949e-03,\n",
      "         7.2018e-03,  1.4826e-02,  2.3742e-02, -6.1321e-02, -2.1383e-02,\n",
      "         3.2315e-02, -7.4180e-02, -1.7717e-03, -9.4235e-03, -2.0061e-02,\n",
      "         2.9836e-02, -3.2257e-02, -5.1903e-02, -3.8832e-03, -1.2342e-02,\n",
      "        -4.5037e-02, -2.9468e-04,  9.2578e-03,  4.0805e-02,  2.7170e-02,\n",
      "         1.3148e-01, -9.6892e-03,  2.6359e-04, -2.9191e-02,  3.0024e-02,\n",
      "        -7.6491e-03, -2.2771e-02,  4.1245e-02, -5.2886e-02, -2.3901e-02,\n",
      "         4.1673e-02, -6.1889e-03,  8.0423e-02, -1.7459e-03, -6.7811e-02,\n",
      "        -3.1979e-02,  6.1105e-02, -1.0903e-02, -6.5751e-02,  4.1530e-02,\n",
      "        -4.1572e-03,  1.7672e-02,  1.8408e-02, -7.3652e-02,  4.2771e-02,\n",
      "        -9.7789e-02, -8.1025e-02,  2.1715e-05, -3.2758e-02, -1.4895e-02,\n",
      "         4.2143e-02, -2.6914e-02,  7.3917e-03,  1.2462e-03,  2.2086e-02,\n",
      "         1.6150e-02,  9.2584e-04, -1.7590e-02, -3.7707e-02, -3.7243e-02,\n",
      "         3.2464e-02, -5.6081e-03,  1.7546e-02, -2.0851e-02,  2.9056e-03,\n",
      "         2.9485e-02,  3.0514e-02, -1.2839e-02,  1.1883e-02,  3.1132e-02,\n",
      "        -2.3356e-02, -2.3855e-02, -4.6466e-02,  2.0564e-02, -7.3287e-03,\n",
      "        -5.9056e-02,  3.5800e-03, -4.2865e-02,  4.5408e-02, -2.0976e-02,\n",
      "         4.8543e-02, -3.4966e-02,  3.0452e-02, -2.5038e-02,  1.5476e-02,\n",
      "         1.0529e-02,  2.6576e-02,  2.1183e-02, -4.7218e-04, -2.9815e-02,\n",
      "         1.3369e-02,  2.0643e-02, -2.9733e-02, -2.7233e-02,  4.2719e-02,\n",
      "        -3.0168e-02, -2.4108e-02, -5.4986e-03, -2.7997e-02, -4.1017e-02,\n",
      "         1.3362e-02, -4.7176e-02,  5.2687e-02, -2.3994e-02,  3.1352e-02,\n",
      "        -1.2392e-02,  2.3217e-02,  4.1690e-02,  1.8759e-02,  1.8617e-01,\n",
      "        -4.9697e-02, -3.0404e-02,  1.4703e-02, -6.3641e-02, -1.0074e-02,\n",
      "        -3.4759e-02, -1.4914e-02,  1.7912e-02, -1.4140e-02,  3.8341e-03,\n",
      "         1.2581e-02,  1.1031e-02, -8.0809e-02,  2.4833e-02,  4.2264e-02,\n",
      "        -3.6010e-02, -2.5384e-02, -7.3533e-02,  3.7758e-02])), ('bert.encoder.layer.4.output.LayerNorm.weight', tensor([0.8668, 0.8899, 0.8750, 0.8563, 0.4625, 0.8463, 0.8538, 0.9078, 0.7802,\n",
      "        0.8874, 0.7867, 0.8647, 0.8303, 0.8347, 0.8735, 0.8928, 0.8353, 0.8433,\n",
      "        0.8503, 0.8572, 0.8445, 0.8349, 0.8529, 0.8696, 0.8561, 0.8480, 0.8573,\n",
      "        0.8609, 0.8771, 0.8718, 0.8847, 0.5578, 0.8414, 0.8726, 0.8702, 0.8482,\n",
      "        0.8361, 0.9034, 0.8475, 0.8701, 0.8668, 0.8531, 0.8758, 0.8224, 0.8534,\n",
      "        0.9043, 0.8845, 0.8530, 0.8559, 0.8972, 0.8720, 0.8729, 0.8642, 0.8899,\n",
      "        0.8710, 0.8769, 0.8948, 0.8521, 0.8278, 0.8842, 0.9111, 0.8909, 0.8958,\n",
      "        0.9080, 0.9148, 0.9090, 0.9772, 0.9095, 0.9344, 0.9398, 0.9187, 0.9247,\n",
      "        0.9152, 0.9284, 0.8350, 0.9360, 0.9146, 0.9639, 0.9376, 0.9178, 0.8715,\n",
      "        0.8779, 0.8493, 0.9581, 0.9282, 0.9284, 0.9740, 0.9771, 0.9323, 0.8930,\n",
      "        0.9677, 0.9115, 0.9712, 0.8675, 0.8955, 0.9333, 1.0033, 0.9749, 0.9474,\n",
      "        0.9625, 0.9978, 0.8779, 0.9768, 0.9737, 0.9156, 0.8710, 0.9383, 0.9647,\n",
      "        0.9875, 0.9454, 0.9731, 0.9630, 0.9458, 0.9350, 0.9789, 0.9586, 0.9580,\n",
      "        0.9431, 0.9618, 0.9260, 0.7884, 0.9804, 0.9743, 0.9549, 0.9565, 0.9577,\n",
      "        0.9884, 0.9404, 0.9562, 0.9468, 0.8913, 0.9433, 0.9828, 0.9694, 0.9529,\n",
      "        0.9999, 0.9152, 0.9911, 0.9218, 0.9712, 0.9463, 0.8534, 0.9962, 0.9460,\n",
      "        0.9387, 0.9577, 0.9718, 0.9747, 0.9633, 0.9423, 0.9807, 0.9361, 0.9704,\n",
      "        0.9846, 0.9178, 0.9525, 0.9299, 0.9401, 0.9065, 0.9013, 0.9786, 1.0006,\n",
      "        0.9355, 0.9667, 0.9955, 0.9004, 0.9975, 0.9247, 0.9349, 0.9798, 0.9692,\n",
      "        0.9210, 0.9752, 0.9387, 1.0001, 1.0041, 0.9885, 0.9358, 0.9804, 0.8646,\n",
      "        0.9831, 0.8602, 0.9458, 0.9368, 0.9134, 0.9591, 0.9870, 1.2746, 0.9762,\n",
      "        0.9217, 0.9445, 0.9576, 0.9543, 0.9597, 0.8975, 0.9438, 0.9484, 0.9620,\n",
      "        0.9752, 0.9399, 0.9057, 0.9386, 0.9613, 0.9642, 0.9491, 0.9601, 0.9571,\n",
      "        0.9575, 0.8478, 0.9283, 0.9517, 0.9847, 0.9179, 0.9778, 0.9733, 0.9844,\n",
      "        0.9653, 0.9727, 0.9375, 0.9973, 0.9192, 0.9773, 0.4143, 0.9470, 0.8771,\n",
      "        0.9557, 0.9442, 0.9178, 0.9471, 0.9608, 0.9126, 0.9681, 0.9455, 0.9663,\n",
      "        0.9170, 1.0010, 0.9180, 0.9869, 0.9465, 0.9613, 0.9414, 0.9744, 0.9631,\n",
      "        0.9363, 0.9907, 0.9025, 0.9363, 0.9705, 0.9529, 0.9743, 0.9734, 0.9313,\n",
      "        0.9530, 0.9174, 0.9633, 0.9202, 0.9555, 0.9699, 0.8481, 0.9711, 0.9909,\n",
      "        0.9522, 0.9499, 0.9556, 0.9655, 0.9233, 0.9609, 1.0087, 0.9200, 0.9522,\n",
      "        0.9769, 0.8211, 0.9337, 0.9613, 0.9509, 0.3846, 0.9666, 0.8955, 0.9581,\n",
      "        0.8148, 0.9381, 0.9285, 0.9422, 0.9000, 0.9463, 0.9579, 0.9189, 0.9022,\n",
      "        0.9581, 0.9418, 0.9612, 0.9874, 0.9403, 0.9462, 0.9630, 0.9667, 0.9860,\n",
      "        0.9670, 0.8879, 0.9029, 0.6631, 0.9028, 0.9595, 0.9668, 0.9601, 0.9489,\n",
      "        0.9801, 0.9512, 0.9224, 0.9418, 0.9574, 0.9151, 0.9811, 0.9550, 0.9085,\n",
      "        0.9696, 0.9536, 0.9069, 0.9319, 0.9940, 0.9933, 0.9611, 0.9791, 0.9554,\n",
      "        0.9724, 0.9224, 0.9410, 0.9783, 0.9654, 0.9406, 0.9496, 0.9433, 0.9298,\n",
      "        0.7618, 0.9141, 0.8876, 0.9411, 0.9356, 0.9635, 0.9721, 0.9974, 0.9730,\n",
      "        0.9562, 0.9156, 0.9478, 0.9768, 0.9486, 0.9637, 0.7873, 0.9618, 0.8634,\n",
      "        0.9719, 0.9520, 0.9269, 0.9530, 0.9520, 0.9442, 0.9597, 0.9636, 0.9515,\n",
      "        0.9560, 0.9365, 0.9402, 0.9019, 0.5019, 0.8985, 0.9813, 0.9333, 0.9163,\n",
      "        0.8771, 0.9399, 0.9579, 0.9619, 0.9725, 0.9404, 0.9318, 0.9532, 0.9257,\n",
      "        0.8801, 0.9280, 0.9484, 0.8933, 0.9890, 0.9470])), ('bert.encoder.layer.4.output.LayerNorm.bias', tensor([ 4.2073e-02,  4.1069e-02,  4.8101e-03,  4.0394e-02, -2.1457e-01,\n",
      "         7.5714e-02,  3.3984e-02,  6.7646e-02, -1.0590e-01,  6.1486e-02,\n",
      "         9.6589e-02, -3.5453e-02, -4.1017e-02,  4.3225e-02,  3.3496e-02,\n",
      "        -9.6867e-03,  2.7154e-02,  3.3836e-03,  4.2797e-02, -1.6399e-02,\n",
      "         6.7818e-02,  6.0112e-02,  1.8837e-01, -3.2275e-02,  6.2778e-02,\n",
      "        -4.1804e-02, -2.9293e-02,  6.7928e-02,  2.7079e-02,  1.9741e-02,\n",
      "         1.0040e-01, -1.4113e-01,  5.3542e-02, -3.6142e-02,  1.7304e-02,\n",
      "         1.0726e-02, -6.5912e-03,  2.8893e-02,  9.1585e-02, -1.7346e-02,\n",
      "         8.0914e-02, -6.8891e-02, -5.8163e-02,  3.3957e-02, -4.1577e-02,\n",
      "         1.9746e-02,  1.0352e-01,  4.0355e-02,  2.2441e-02, -1.6760e-02,\n",
      "         1.2616e-01, -1.1641e-01,  2.0341e-02,  7.0397e-02,  2.3987e-02,\n",
      "         7.8288e-02,  2.3590e-02,  7.4075e-02,  5.6923e-02, -5.2618e-02,\n",
      "         2.2192e-02,  2.0908e-01, -2.8868e-02,  2.3673e-02, -1.8371e-03,\n",
      "         9.8599e-02, -4.4196e-02,  5.6423e-02, -2.3151e-02,  1.0725e-02,\n",
      "         2.9824e-02,  9.3820e-02,  2.3062e-02,  5.1799e-02,  1.7792e-01,\n",
      "        -5.7411e-03, -1.3437e-02, -3.3366e-02,  8.7372e-02,  9.6089e-02,\n",
      "        -1.4283e-01,  4.1142e-02,  6.4615e-02, -6.7396e-02,  3.5548e-02,\n",
      "         4.6606e-02,  5.0551e-02, -3.3877e-02,  3.3020e-02,  6.5782e-03,\n",
      "         1.6246e-02, -9.6668e-03,  1.7768e-02, -9.9773e-02,  5.9573e-02,\n",
      "        -8.7120e-02,  8.6598e-02, -3.1567e-02,  9.4000e-02,  1.7317e-02,\n",
      "         7.7398e-02, -2.9920e-02,  2.5591e-03,  3.4712e-02,  8.3932e-02,\n",
      "        -3.8951e-02,  2.3642e-02, -5.8632e-03,  2.7191e-02, -2.4762e-02,\n",
      "         3.2907e-02, -5.1822e-02,  4.8056e-02,  9.6343e-04, -4.1961e-02,\n",
      "         1.8092e-03,  2.5794e-02,  8.6221e-02,  5.3269e-02, -5.6969e-02,\n",
      "        -3.2719e-01, -1.6810e-03, -1.2958e-02,  8.3906e-03,  2.6126e-02,\n",
      "         2.5081e-02,  6.6628e-02,  5.7235e-02,  4.8323e-02,  3.3920e-02,\n",
      "        -2.5546e-02,  4.4807e-03, -9.9295e-03,  6.3502e-02,  3.9680e-02,\n",
      "         1.3521e-02,  1.3658e-02, -4.8549e-02,  6.3985e-02,  9.2843e-03,\n",
      "         7.8900e-02, -2.6527e-02,  2.9939e-02,  3.6267e-02,  7.4821e-02,\n",
      "         2.5398e-02,  4.2351e-02, -9.0087e-03,  1.2412e-02,  4.0201e-02,\n",
      "        -2.0261e-02, -9.3419e-03, -2.2931e-02,  1.0788e-02, -3.3208e-02,\n",
      "         2.4936e-02, -3.3574e-02, -2.9342e-02,  4.4288e-02, -4.8633e-02,\n",
      "         4.1425e-02, -3.1777e-02,  6.0143e-02, -8.0101e-02,  2.5253e-02,\n",
      "         3.2026e-02,  6.3549e-02,  9.6779e-03,  4.1686e-02,  8.4376e-03,\n",
      "         4.5628e-02,  5.7408e-02,  3.5260e-02, -7.7768e-02,  7.0914e-02,\n",
      "         5.3386e-02,  1.2889e-02,  5.7762e-02,  1.8590e-02, -9.7816e-02,\n",
      "         4.3052e-02,  4.5631e-02,  5.7018e-02, -4.4698e-02,  8.0229e-02,\n",
      "         5.6757e-02,  5.2881e-02,  4.5928e-02,  4.3734e-02, -7.1242e-02,\n",
      "        -1.3928e-02,  7.7690e-03,  2.1930e-02, -1.0244e-02,  2.7608e-03,\n",
      "        -9.2811e-02, -4.3232e-02,  5.5073e-03, -6.4639e-03, -2.6922e-02,\n",
      "         1.0272e-01,  8.6679e-02, -2.6699e-02, -3.1550e-03,  2.0064e-02,\n",
      "         4.9187e-02, -4.8219e-02,  1.4601e-02,  1.4498e-01,  2.0821e-02,\n",
      "         8.5519e-02,  5.6972e-02,  6.8965e-02, -5.2500e-03,  5.9640e-02,\n",
      "        -9.0792e-03,  2.1036e-02,  5.0419e-02,  5.1734e-02,  1.0325e-02,\n",
      "         7.2588e-02,  1.4772e-02,  1.9131e-01,  5.5011e-02,  9.7451e-02,\n",
      "         7.0943e-03,  4.4863e-02, -8.0846e-02,  7.0814e-02,  4.5529e-02,\n",
      "         1.9068e-01,  3.6407e-02,  5.9454e-02,  5.4728e-02,  1.0109e-01,\n",
      "         6.7302e-02, -1.0268e-02,  2.0909e-02,  6.0346e-02, -3.1846e-03,\n",
      "         7.3779e-02,  3.2376e-02,  5.2527e-02, -8.7040e-03, -2.3126e-03,\n",
      "        -1.9230e-02,  1.3332e-01,  4.7172e-03,  1.0739e-01,  2.2530e-03,\n",
      "        -1.7292e-02,  2.7106e-03,  4.5180e-02,  7.3288e-02,  5.3876e-02,\n",
      "        -7.8281e-02,  1.2905e-02,  5.3607e-03,  4.4214e-02,  4.0349e-02,\n",
      "         4.8522e-02, -5.8882e-03,  2.8844e-02,  1.7305e-02,  1.7461e-02,\n",
      "        -4.4345e-02, -1.0469e-02,  2.6477e-03,  9.4664e-02, -1.9064e-02,\n",
      "         7.1029e-03, -1.5471e-01,  2.2840e-02, -1.8722e-02,  8.9632e-03,\n",
      "        -2.7497e-01,  3.6745e-02, -1.1165e-01,  5.6760e-03, -1.1191e-01,\n",
      "         8.6730e-02,  4.9465e-02,  5.5445e-02,  1.5722e-02,  1.3967e-02,\n",
      "         8.9724e-03,  7.4831e-02, -3.9154e-02,  3.1850e-02, -6.7253e-02,\n",
      "         2.9259e-02,  6.0192e-02,  7.6176e-02, -6.3597e-02,  1.1311e-01,\n",
      "        -4.7756e-02, -8.6084e-03, -2.9523e-03, -2.6724e-02, -2.9930e-03,\n",
      "         1.2667e-01, -2.4017e-02, -8.0860e-03,  1.3141e-02,  4.3405e-02,\n",
      "         5.3508e-02, -2.5780e-03, -3.2937e-02,  2.6142e-02,  2.1424e-02,\n",
      "         5.5018e-02,  9.1410e-03, -9.3613e-03, -1.4999e-02,  3.9422e-02,\n",
      "         5.7519e-02,  5.8964e-02, -3.6026e-02, -8.0923e-02,  2.9775e-02,\n",
      "         2.1161e-02,  1.2614e-01,  6.6655e-02,  1.1523e-03,  6.3573e-02,\n",
      "        -6.7019e-02,  8.8214e-02, -2.4053e-02,  2.2498e-02,  1.9000e-02,\n",
      "         7.1396e-02, -1.8101e-02,  2.5179e-02, -2.2664e-01, -7.3219e-03,\n",
      "        -3.1755e-04,  3.6800e-02,  9.6016e-03, -3.9085e-02, -4.2757e-02,\n",
      "        -1.3339e-02,  8.6121e-04,  8.4805e-02, -7.0662e-02,  1.1606e-01,\n",
      "         2.2216e-02,  7.2198e-02,  1.2559e-02,  1.7564e-01,  3.0793e-02,\n",
      "         1.8118e-01, -2.3554e-02,  1.9163e-02,  7.7431e-03, -5.6565e-02,\n",
      "        -1.7359e-02,  3.5756e-02, -2.4959e-02,  1.3310e-02,  3.2697e-02,\n",
      "         3.2882e-02,  1.7072e-02,  3.7084e-02, -2.2866e-02, -3.4783e-01,\n",
      "         4.1117e-02, -5.2427e-02, -1.7968e-02, -5.4710e-02,  7.5364e-02,\n",
      "         1.2175e-02,  6.4021e-02, -5.5029e-03, -1.5905e-02,  3.5429e-02,\n",
      "         3.5881e-02,  1.4934e-03, -2.4826e-04,  1.9438e-01, -5.4126e-02,\n",
      "        -5.7481e-02, -1.5065e-03, -2.8086e-02,  7.0392e-02])), ('bert.encoder.layer.5.attention.self.query.weight', tensor([[ 2.8599e-02,  6.0008e-02, -1.0077e-01,  ...,  1.7887e-01,\n",
      "         -5.9511e-03, -1.2806e-01],\n",
      "        [-1.0860e-01, -7.4575e-02,  1.0395e-01,  ..., -1.1517e-01,\n",
      "          9.6210e-02, -4.4038e-02],\n",
      "        [-2.7322e-02, -1.3957e-01,  5.8772e-03,  ..., -3.0017e-02,\n",
      "          2.5341e-05,  3.9870e-02],\n",
      "        ...,\n",
      "        [-6.3381e-02, -8.2701e-04, -6.1833e-02,  ..., -5.5712e-02,\n",
      "         -8.3705e-02, -4.5913e-03],\n",
      "        [ 1.0373e-02,  9.6759e-03, -3.9198e-02,  ..., -3.8453e-02,\n",
      "         -7.3408e-02, -1.7960e-02],\n",
      "        [-1.6195e-02,  2.8169e-02, -5.3442e-03,  ...,  5.7982e-02,\n",
      "         -2.2934e-02,  4.6177e-03]])), ('bert.encoder.layer.5.attention.self.query.bias', tensor([-4.6809e-02, -5.5637e-03, -5.6146e-03, -5.3351e-02, -9.9362e-03,\n",
      "         1.3521e-02, -1.1701e-02, -2.7724e-02, -1.1706e-02, -7.6704e-02,\n",
      "        -3.5321e-02, -6.3558e-03, -4.8750e-02,  4.8983e-02, -5.0828e-02,\n",
      "         1.2039e-02,  8.0525e-02,  8.9337e-02, -6.4377e-02, -8.3392e-02,\n",
      "         2.8131e-02,  1.0096e-02,  4.6082e-02, -2.1008e-02, -2.7741e-02,\n",
      "        -1.0176e-02,  7.3189e-02, -1.0425e-01, -4.8838e-02,  3.4087e-02,\n",
      "        -3.0970e-03,  1.6296e-03,  3.6526e-02,  1.0457e-02,  5.4141e-02,\n",
      "         7.3104e-02,  1.5282e-02, -3.0434e-02, -1.7894e-02,  8.9811e-03,\n",
      "         5.5028e-02,  9.5891e-03,  4.7068e-03,  6.3307e-02, -3.3111e-02,\n",
      "        -5.3056e-02, -6.3932e-02, -2.4338e-02, -9.5432e-03, -5.6527e-02,\n",
      "        -1.2048e-02,  1.3560e-03,  4.9483e-02,  2.6650e-02, -1.2442e-02,\n",
      "        -6.9203e-02, -1.6900e-02,  6.8648e-03,  6.5018e-02,  8.1275e-03,\n",
      "         9.4740e-02, -3.8896e-02, -5.8579e-02,  6.6560e-02, -1.2217e-01,\n",
      "        -2.8864e-01, -7.2091e-03, -4.1127e-02, -3.6595e-01, -1.0015e-01,\n",
      "        -2.5186e-02,  1.0469e-01,  1.0480e-01,  2.3170e-01,  9.7841e-02,\n",
      "        -2.0524e-02,  3.6050e-01,  8.6447e-02, -3.2151e-01, -3.4240e-01,\n",
      "        -8.2147e-03, -1.0975e-02,  4.0280e-01,  7.2431e-02,  2.6405e-01,\n",
      "        -2.4579e-01, -1.1900e-01, -9.3714e-03, -8.5027e-02, -3.0045e-01,\n",
      "         8.2640e-02, -2.1563e-03,  6.5009e-02,  4.0905e-02,  7.8003e-02,\n",
      "         1.6474e-01,  9.1960e-02,  3.7215e-02, -2.2214e-02, -1.3901e-01,\n",
      "        -1.9033e-02, -1.8635e-01, -1.4655e-01, -2.1841e-01, -4.6992e-02,\n",
      "        -1.2036e-01, -7.1423e-02, -1.0034e-01, -8.1077e-02, -4.6640e-02,\n",
      "        -1.7147e-01, -1.0382e-04,  7.3719e-02,  9.7741e-02, -1.4192e-02,\n",
      "        -2.0718e-02, -3.6618e-03,  1.8073e-02, -1.3153e-01, -1.2852e-01,\n",
      "        -8.3783e-02, -6.7728e-03, -1.2236e-01,  1.6534e-01, -1.9110e-01,\n",
      "        -7.7033e-02,  1.2627e-01, -6.5798e-03, -1.0596e-01, -4.0432e-01,\n",
      "        -3.6290e-02,  8.1416e-03, -1.2582e-01, -1.5292e-01,  3.3023e-01,\n",
      "        -3.6638e-03,  1.9363e-01, -3.4507e-01, -2.7571e-01, -2.9605e-01,\n",
      "         1.0009e-01, -2.6119e-01, -1.0786e-01, -2.2049e-01,  1.8588e-01,\n",
      "         1.5966e-01, -2.4208e-01, -5.9968e-02, -7.3866e-02,  4.8492e-02,\n",
      "        -7.8007e-02, -2.8908e-01,  2.0727e-01, -4.4985e-02, -3.0196e-01,\n",
      "        -2.9827e-01, -2.4124e-01, -2.6307e-01,  2.4209e-01,  6.8159e-03,\n",
      "         6.5430e-02, -3.8301e-02,  1.5826e-01, -4.3235e-02,  6.1261e-02,\n",
      "        -1.8417e-01, -1.6055e-01,  7.1641e-03,  2.8120e-02, -6.9332e-02,\n",
      "        -7.4621e-02, -1.0967e-01, -4.5805e-02, -5.0131e-02,  1.0474e-01,\n",
      "        -5.8940e-02, -8.3695e-02,  1.2727e-01,  2.3954e-02, -1.3493e-01,\n",
      "         1.3201e-01, -1.8204e-02,  9.3253e-03, -1.3574e-01,  1.6156e-03,\n",
      "         4.4911e-02, -5.8655e-02, -5.2137e-02, -3.8675e-02, -1.1210e-01,\n",
      "        -3.1991e-02, -1.2060e-02, -2.3798e-02, -2.6661e-03,  2.0005e-02,\n",
      "         2.1751e-02,  2.1896e-02, -2.6620e-02, -2.6418e-02,  1.5039e-02,\n",
      "        -3.1288e-02,  1.4065e-02,  9.9822e-03, -1.8673e-02, -1.6653e-02,\n",
      "        -4.1939e-02,  3.0512e-02,  1.2464e-02,  1.5800e-02,  2.6583e-02,\n",
      "         2.1270e-02,  1.4251e-02, -1.1067e-02, -2.6896e-02, -1.0089e-02,\n",
      "        -3.5198e-02, -3.9042e-02, -2.3587e-02, -1.6027e-02,  2.5384e-02,\n",
      "         2.2857e-02, -6.9981e-03,  5.0646e-03, -1.0475e-02,  5.0996e-02,\n",
      "         1.7646e-02, -4.5125e-02,  4.1206e-02,  2.4659e-02,  6.5898e-02,\n",
      "        -4.7420e-02,  1.6068e-02, -1.9233e-02, -1.8321e-03,  4.1076e-02,\n",
      "        -1.8618e-02,  2.0127e-02,  2.9776e-02, -4.9961e-02,  8.1817e-02,\n",
      "         2.2350e-02, -1.4218e-02,  1.9279e-02, -6.3302e-02,  5.2109e-02,\n",
      "        -1.9450e-02, -2.0579e-02,  4.9498e-02,  2.9222e-02,  1.0515e-02,\n",
      "         3.1500e-02, -9.8276e-02, -6.9296e-03,  3.0935e-02,  6.0502e-02,\n",
      "        -5.7188e-03, -5.9266e-03, -8.8589e-03, -1.7783e-02, -7.0235e-03,\n",
      "        -9.2401e-03,  7.5486e-03, -1.5642e-02, -1.7127e-02, -9.2655e-04,\n",
      "         1.5863e-02,  1.2148e-02, -1.0196e-04,  6.1395e-03, -2.5268e-02,\n",
      "         7.7279e-03, -8.7880e-03, -2.2010e-02, -2.4589e-02, -1.9475e-02,\n",
      "        -1.0016e-02,  1.3259e-02, -5.5756e-03,  1.7306e-02,  1.2118e-02,\n",
      "        -2.1569e-02,  1.0821e-02,  9.3137e-03, -2.7972e-03, -3.9649e-02,\n",
      "        -1.6556e-02,  1.9189e-02, -1.7712e-02,  1.1089e-01, -9.9635e-02,\n",
      "         9.3162e-02,  4.7429e-02, -2.2791e-02, -1.6775e-01, -4.1677e-02,\n",
      "        -7.1839e-02,  2.9346e-02,  2.0101e-02, -8.7803e-02,  1.3085e-02,\n",
      "         1.0883e-02,  1.8230e-02,  3.0573e-02,  7.7905e-02,  6.8159e-02,\n",
      "        -1.1879e-02, -2.4954e-02, -2.9991e-03,  7.6322e-02, -1.2226e-01,\n",
      "         3.8316e-02, -1.3303e-02, -1.7086e-02,  2.1630e-02, -7.7190e-02,\n",
      "         9.5801e-02, -1.0498e-01, -9.2659e-02,  7.1391e-02,  1.1214e-02,\n",
      "         1.3356e-01, -1.0564e-01, -2.7447e-01, -9.0403e-02,  5.3969e-02,\n",
      "         1.7924e-01,  3.1617e-01, -1.8984e-01,  2.2819e-01, -2.0830e-01,\n",
      "         2.0924e-01,  1.9504e-02, -2.7314e-02,  2.3768e-01,  9.4441e-02,\n",
      "        -2.8095e-02, -6.9609e-02,  9.9874e-02,  1.4986e-01,  4.8232e-01,\n",
      "         3.6555e-01, -1.8516e-01,  5.2790e-02, -8.8694e-02, -8.4550e-02,\n",
      "         3.1772e-01,  3.8474e-01, -2.1583e-01,  1.0608e-01, -3.0911e-01,\n",
      "         2.1975e-01,  2.4212e-01, -9.3221e-02, -1.6554e-02, -3.1372e-02,\n",
      "         9.7294e-03,  5.7810e-02,  6.3938e-02, -5.3031e-02, -3.0220e-02,\n",
      "        -1.0553e-02,  2.4423e-02, -7.7819e-03,  5.2002e-02,  1.7152e-02,\n",
      "        -5.7956e-02,  3.4639e-02,  1.7544e-02,  9.6271e-03, -9.8805e-03,\n",
      "         4.8460e-02,  5.5013e-02, -7.2420e-02,  1.5928e-02,  7.9410e-03,\n",
      "         7.8720e-02, -4.6373e-02,  2.8183e-02,  6.1234e-02, -4.3851e-02,\n",
      "         3.4998e-02, -1.3095e-02,  5.3436e-02, -3.3840e-02])), ('bert.encoder.layer.5.attention.self.key.weight', tensor([[ 0.0288, -0.0469, -0.0545,  ...,  0.0783, -0.0499, -0.0527],\n",
      "        [-0.0056, -0.0270,  0.0590,  ..., -0.0402,  0.0955, -0.0628],\n",
      "        [ 0.1279, -0.0546, -0.0067,  ..., -0.1163, -0.0810,  0.0723],\n",
      "        ...,\n",
      "        [ 0.0424,  0.0082, -0.0190,  ...,  0.0199,  0.1300,  0.0587],\n",
      "        [-0.0307, -0.0321,  0.0041,  ...,  0.0350, -0.0490, -0.0964],\n",
      "        [ 0.0721, -0.0818,  0.0238,  ..., -0.0107, -0.1517,  0.0122]])), ('bert.encoder.layer.5.attention.self.key.bias', tensor([-2.0001e-02,  2.4685e-04, -1.3907e-02,  2.5913e-03,  2.7583e-03,\n",
      "         3.2664e-03, -6.0674e-03, -1.9436e-04,  1.7205e-03, -1.0566e-02,\n",
      "         2.6008e-04, -1.0471e-03, -1.8199e-02, -6.3933e-03, -1.8965e-03,\n",
      "         6.7617e-03,  1.9421e-02,  1.0568e-02, -8.7043e-03, -2.9573e-05,\n",
      "        -1.5484e-02, -4.0215e-03,  6.0614e-03, -2.9614e-03, -7.0307e-03,\n",
      "         2.3672e-03,  1.0969e-02, -1.4219e-02,  1.6928e-03,  8.0563e-04,\n",
      "         1.1124e-02, -6.8640e-03, -1.9461e-03,  1.4517e-03,  2.5860e-03,\n",
      "        -2.6866e-03,  9.8208e-04,  1.3517e-03,  1.1200e-03,  6.1921e-04,\n",
      "        -7.7334e-04, -7.0099e-04, -1.9292e-03,  4.3310e-04,  1.7761e-03,\n",
      "        -2.1042e-03,  1.0786e-03, -7.6470e-05,  1.7679e-04,  3.6720e-04,\n",
      "        -1.4452e-04,  3.5894e-03, -2.5782e-03, -2.4816e-04, -4.7164e-04,\n",
      "         1.0568e-03, -1.1135e-03,  5.6515e-03, -2.0443e-04,  2.1533e-04,\n",
      "        -2.1356e-03, -3.7617e-05,  9.5578e-04, -6.7912e-04, -1.1468e-02,\n",
      "        -4.9219e-03, -8.4768e-04,  1.0665e-02, -7.4138e-03, -4.9005e-04,\n",
      "         5.0371e-03,  8.0300e-03,  8.2038e-03,  8.8165e-03,  1.3225e-03,\n",
      "         6.4782e-03,  8.9554e-03,  1.3093e-03, -4.1623e-03, -1.2488e-02,\n",
      "        -5.4370e-03,  2.8636e-03,  2.0121e-02, -3.1794e-03,  1.7506e-02,\n",
      "        -1.3385e-02,  2.3487e-03,  3.5315e-03, -4.1293e-03, -6.6022e-03,\n",
      "        -5.2709e-03,  2.5964e-03, -4.0547e-03,  5.9969e-03,  2.4899e-03,\n",
      "         1.8073e-03, -1.6378e-03,  8.2709e-05, -8.5889e-04, -1.4484e-03,\n",
      "        -8.4487e-04,  1.6212e-03,  2.9693e-04,  6.6653e-04,  2.3023e-03,\n",
      "         1.3125e-03, -9.2916e-04, -1.0139e-04, -3.9677e-04,  6.3541e-03,\n",
      "        -4.9001e-05, -1.9700e-03, -1.5027e-03, -1.6327e-03, -1.5529e-03,\n",
      "         3.3880e-04, -2.2337e-03, -2.7211e-03, -2.0219e-04, -1.0985e-04,\n",
      "        -1.5331e-03,  1.9136e-03,  6.3497e-05,  1.7085e-03,  1.3343e-03,\n",
      "        -1.3945e-03,  2.9594e-03,  2.1315e-03, -5.5759e-03, -1.0410e-02,\n",
      "        -1.3295e-04, -4.6221e-03,  9.9156e-03, -1.3126e-02,  8.8443e-03,\n",
      "        -5.3946e-03,  1.0307e-02, -9.8524e-03, -2.5412e-03, -4.7565e-03,\n",
      "         3.4832e-04, -1.3299e-02,  3.1825e-03, -3.4307e-03,  5.2784e-03,\n",
      "         7.7773e-03, -6.8854e-03,  6.4816e-03, -1.8121e-03, -4.4572e-03,\n",
      "        -1.1958e-02,  4.3632e-03,  1.6786e-03,  3.2357e-03, -2.5503e-03,\n",
      "        -5.1743e-03, -5.6103e-03, -4.4082e-04,  9.5049e-03, -6.6743e-04,\n",
      "         1.5768e-03,  5.9415e-04, -1.1150e-03,  2.0420e-03, -3.2847e-04,\n",
      "        -4.2211e-03, -7.2900e-04,  2.0800e-04,  5.8219e-04, -1.8671e-03,\n",
      "        -2.1200e-04, -2.1372e-03, -1.6569e-03, -3.8701e-04,  1.8558e-04,\n",
      "        -3.0063e-04, -2.6858e-03,  1.3932e-03,  2.5800e-03,  8.2080e-05,\n",
      "         7.4714e-04,  2.0047e-04,  8.9885e-04, -3.7557e-04, -8.2590e-04,\n",
      "         1.4415e-03,  3.1093e-04,  1.7729e-03, -1.1067e-05,  1.9932e-04,\n",
      "        -1.4276e-03, -2.4480e-03, -2.1344e-03, -7.5479e-04, -8.4973e-04,\n",
      "        -9.2048e-05,  5.9712e-04, -8.0969e-04,  3.3570e-03, -4.5456e-03,\n",
      "        -1.7006e-04, -2.1709e-03, -1.7563e-03, -1.1565e-03,  8.0063e-04,\n",
      "        -1.2039e-05,  1.7752e-03, -6.5706e-03,  2.5649e-03, -6.1203e-04,\n",
      "        -5.5969e-04, -5.5802e-04, -6.3613e-04,  6.2609e-04,  5.1061e-04,\n",
      "         2.2257e-04, -5.7771e-04,  1.1620e-03, -2.0974e-04,  3.8899e-04,\n",
      "        -2.1321e-03,  1.4630e-04, -2.3585e-04,  2.0627e-04,  2.2926e-04,\n",
      "         4.4654e-04,  2.4416e-03, -1.7219e-03, -1.9249e-04, -1.8085e-03,\n",
      "         8.8593e-04, -1.1985e-03,  4.1338e-04,  1.4568e-04, -7.5503e-03,\n",
      "        -5.3158e-04,  4.7219e-05, -4.9731e-05,  1.2849e-03, -3.2171e-03,\n",
      "        -2.7904e-03,  1.4012e-03, -2.3857e-03,  2.5944e-03, -3.3378e-03,\n",
      "         3.4096e-03,  3.1199e-03, -1.5251e-03, -1.6309e-03,  6.6464e-04,\n",
      "        -2.0980e-03,  3.0802e-03,  3.6671e-03, -2.7324e-03, -1.6632e-03,\n",
      "        -1.5406e-03, -9.8913e-04,  4.0470e-04,  1.2379e-03,  1.4651e-03,\n",
      "        -4.9295e-04, -1.0625e-03, -2.7598e-04, -4.8968e-04, -6.9735e-04,\n",
      "        -2.0789e-03, -5.5483e-05,  1.7551e-03,  4.5471e-04, -1.3156e-03,\n",
      "        -8.9237e-04,  8.4852e-04, -9.0210e-04,  7.4631e-04, -1.3880e-03,\n",
      "         2.4647e-04, -6.5548e-04,  1.2276e-03,  1.2515e-03,  1.6115e-03,\n",
      "        -5.2636e-04, -3.6571e-04,  3.3931e-04,  2.3550e-04,  8.6638e-03,\n",
      "         1.0981e-03, -2.1462e-03, -1.5593e-03, -1.1578e-03,  1.1997e-05,\n",
      "        -9.7069e-04, -1.1413e-03, -1.2436e-03,  1.6094e-03, -4.6991e-04,\n",
      "        -3.2784e-04,  1.3253e-03, -4.6461e-04, -1.8398e-03,  1.8899e-03,\n",
      "         7.0720e-05, -1.8766e-03, -1.6663e-03, -4.8646e-04, -9.9102e-05,\n",
      "        -4.2132e-03,  2.7380e-03,  8.4017e-04,  4.0900e-04, -1.0409e-03,\n",
      "        -1.2210e-04, -1.5086e-03, -9.6181e-04,  2.0140e-03,  4.5216e-04,\n",
      "        -6.7710e-04,  8.5328e-04,  3.2622e-04, -1.7255e-03,  1.6816e-03,\n",
      "        -2.6447e-04, -7.0636e-03, -8.7249e-03, -2.5722e-04, -7.9475e-04,\n",
      "         5.7843e-03,  1.1090e-02, -4.7289e-03,  2.9521e-03, -1.1134e-02,\n",
      "         3.2173e-04, -5.8656e-03, -3.5063e-03,  3.2597e-03,  5.3703e-03,\n",
      "         1.6681e-03,  5.7812e-03,  1.2301e-03,  8.8573e-04,  1.6927e-02,\n",
      "         1.2512e-02,  4.5055e-04,  6.0724e-03, -5.5952e-03, -1.2636e-03,\n",
      "         1.3076e-02,  1.5565e-02, -6.6629e-04,  2.0395e-03, -1.0387e-02,\n",
      "         1.1839e-02,  1.2053e-02, -2.2345e-03,  9.1456e-04,  5.8681e-04,\n",
      "        -6.9729e-04,  6.9049e-04, -3.4013e-04, -1.4024e-03, -8.8871e-05,\n",
      "         1.4755e-03, -1.2975e-03,  9.0324e-04,  5.2878e-04, -1.8221e-03,\n",
      "        -1.0806e-03,  7.7760e-04, -2.1543e-04,  6.2181e-04, -3.2243e-04,\n",
      "         3.6578e-04, -8.1112e-04, -8.0302e-04, -1.5958e-03, -1.0845e-03,\n",
      "        -1.1488e-03,  1.2515e-03, -1.0564e-03,  2.4727e-04,  8.1519e-04,\n",
      "         1.3296e-03,  1.5794e-03, -3.6083e-04, -1.0828e-03])), ('bert.encoder.layer.5.attention.self.value.weight', tensor([[-0.0321,  0.0315, -0.0554,  ...,  0.0318,  0.0914, -0.0158],\n",
      "        [ 0.0129, -0.0216, -0.0474,  ..., -0.0498,  0.0008, -0.0743],\n",
      "        [-0.0293, -0.0754, -0.0199,  ...,  0.0312,  0.0244,  0.0103],\n",
      "        ...,\n",
      "        [ 0.0530, -0.0123, -0.0180,  ...,  0.0265,  0.0700, -0.0226],\n",
      "        [ 0.0328,  0.0773,  0.0403,  ...,  0.0452, -0.0287, -0.0474],\n",
      "        [-0.0172, -0.0433,  0.0537,  ...,  0.0272,  0.0338,  0.0290]])), ('bert.encoder.layer.5.attention.self.value.bias', tensor([-8.8461e-02,  7.0008e-02, -6.1714e-02,  4.3417e-02,  6.6184e-02,\n",
      "        -2.3269e-03,  4.8292e-02,  1.1583e-02,  1.0154e-01,  2.0333e-02,\n",
      "         3.3069e-02,  6.6547e-02,  3.9158e-02, -1.0090e-01, -4.6934e-02,\n",
      "        -5.1283e-02, -7.1993e-02,  2.3723e-02,  8.9174e-02,  2.6493e-02,\n",
      "        -4.5194e-02, -8.1284e-02,  1.1461e-02,  9.2582e-02,  2.0024e-02,\n",
      "        -4.8368e-02,  2.3826e-03, -9.0172e-02, -5.6651e-02,  5.6835e-02,\n",
      "         1.3222e-01, -1.9625e-02, -2.1667e-03,  3.7137e-02, -6.4099e-03,\n",
      "         1.5879e-03, -1.1818e-02,  2.8192e-03,  1.5128e-02, -6.4103e-03,\n",
      "        -1.2208e-02,  3.3575e-03, -1.4115e-02, -4.6406e-02,  2.0280e-02,\n",
      "        -2.8112e-03, -3.7396e-03,  2.9308e-02, -8.1419e-03,  8.0741e-03,\n",
      "         6.3130e-03,  1.4010e-02, -8.8258e-03,  1.1375e-02,  5.8881e-03,\n",
      "         6.4245e-04, -6.5296e-04, -5.4567e-03,  4.0040e-03,  1.2943e-02,\n",
      "        -1.0315e-02, -9.8263e-03,  6.4600e-03,  1.2088e-02, -8.0292e-03,\n",
      "        -2.6951e-02, -3.1014e-03, -1.9041e-02, -1.8700e-02, -2.0548e-02,\n",
      "        -7.7374e-03, -1.0522e-02,  2.0102e-02, -6.3700e-03, -1.5140e-02,\n",
      "        -2.5319e-02, -7.0840e-03,  1.6714e-02, -2.2174e-02, -3.0400e-02,\n",
      "         3.5901e-02, -5.4552e-03, -2.1862e-02, -1.4786e-02,  1.2735e-02,\n",
      "         3.0379e-02, -3.4290e-03,  4.9599e-02, -1.7728e-02,  7.1154e-03,\n",
      "         2.9657e-02,  1.0450e-02, -3.5708e-03, -2.3426e-02,  2.6925e-03,\n",
      "         2.6245e-02,  4.3482e-03,  2.0086e-03, -2.3641e-02,  1.1069e-02,\n",
      "        -8.2403e-03,  3.4852e-02,  5.8584e-03,  3.0226e-03, -1.1249e-02,\n",
      "        -2.7126e-03,  1.7189e-02, -5.9124e-03, -1.2861e-02,  3.6185e-03,\n",
      "         2.4903e-02,  2.0521e-02,  2.0309e-03, -2.6437e-02, -7.8021e-03,\n",
      "         2.6003e-03,  5.0598e-03,  4.1115e-02, -7.5443e-03,  3.1625e-03,\n",
      "         2.5917e-02,  3.6670e-03,  1.0934e-02, -3.8848e-02,  6.5989e-03,\n",
      "         6.1862e-03, -1.7501e-02, -8.2887e-03,  1.4837e-02, -2.8318e-02,\n",
      "        -1.5794e-02, -2.3271e-02, -6.3729e-03, -4.5837e-03, -4.9253e-03,\n",
      "        -2.2711e-02,  2.1790e-02, -3.3569e-02,  1.2170e-03, -1.7605e-02,\n",
      "        -3.5167e-02,  5.0838e-02,  9.2314e-04,  7.3947e-03,  2.0919e-02,\n",
      "         2.4185e-02,  4.7986e-03, -3.6703e-02,  1.0602e-02, -1.2983e-02,\n",
      "        -5.1796e-03, -3.0834e-02,  4.1412e-03, -1.4971e-02, -2.9496e-02,\n",
      "        -6.8502e-03, -3.3118e-03, -4.8545e-03,  1.6334e-02,  2.9882e-02,\n",
      "        -1.8098e-03, -1.4396e-02, -2.1407e-02, -7.8059e-03, -1.6788e-02,\n",
      "         3.0138e-02, -4.0112e-02, -5.8374e-03,  1.3440e-04, -2.9745e-02,\n",
      "         1.4250e-02,  5.6662e-03,  1.4348e-02,  1.9182e-03, -1.9062e-03,\n",
      "         2.1125e-02,  8.9329e-03,  6.4431e-04,  2.9588e-03,  2.4000e-03,\n",
      "        -1.1874e-02,  1.5509e-02,  6.9600e-03, -1.1745e-02,  5.1990e-03,\n",
      "         1.6506e-02, -7.6396e-03, -5.2694e-02, -1.3972e-02,  9.4727e-03,\n",
      "         2.1675e-04, -1.3599e-02, -4.5247e-04,  1.5079e-03, -1.8650e-02,\n",
      "         1.9514e-02, -7.1398e-03, -6.0542e-03,  1.7919e-02,  1.5333e-02,\n",
      "        -5.1466e-03,  4.1944e-03, -2.2945e-02,  1.4893e-03,  5.2894e-03,\n",
      "         4.5650e-03, -2.1371e-02,  2.0119e-02,  1.3357e-02,  1.4793e-02,\n",
      "         2.1618e-03, -1.3698e-02, -1.6208e-02, -8.0340e-03, -1.1293e-02,\n",
      "         9.4774e-03, -8.1961e-03,  9.4950e-03,  5.5753e-03,  6.1334e-03,\n",
      "        -2.5737e-02, -1.7061e-03,  5.5981e-03,  1.7636e-02,  2.5926e-03,\n",
      "         4.5475e-02,  1.8221e-04,  1.3662e-02,  3.4820e-02, -3.8306e-02,\n",
      "        -8.4032e-03,  6.8444e-03,  9.3789e-03, -1.3898e-02, -2.4880e-02,\n",
      "        -8.6712e-03,  2.7229e-02, -2.5569e-02, -1.0150e-02,  2.8059e-02,\n",
      "         2.1555e-02,  3.9321e-03,  1.4194e-02, -7.6186e-03, -8.3274e-03,\n",
      "        -6.6307e-03, -1.2424e-02,  6.5244e-04,  3.2751e-02, -6.7141e-03,\n",
      "         2.0422e-02, -8.0250e-03, -1.8141e-02, -1.3533e-02,  8.5615e-03,\n",
      "        -2.6366e-02,  4.4525e-03, -1.5161e-02, -1.2716e-02,  1.9453e-02,\n",
      "        -2.6096e-02,  6.7246e-03,  2.6459e-04,  1.3994e-02, -6.5187e-03,\n",
      "        -8.8819e-03, -1.4308e-02, -9.2925e-03,  3.2647e-03,  1.2719e-02,\n",
      "         5.8796e-03, -1.5424e-02, -5.1003e-03,  6.6934e-04,  5.7021e-03,\n",
      "        -2.0302e-03,  2.0400e-03,  3.4778e-03,  1.5202e-02,  2.0715e-02,\n",
      "         1.3355e-02,  2.1972e-02, -4.6619e-03,  7.0441e-03, -9.2681e-03,\n",
      "        -1.1726e-02, -1.2942e-02,  3.0563e-02,  2.7329e-02,  3.6652e-02,\n",
      "         2.3986e-02, -7.8227e-03,  2.3117e-02, -4.2789e-03, -1.1069e-02,\n",
      "        -6.9638e-02, -1.1826e-02,  6.4486e-03, -1.0059e-02,  4.1919e-02,\n",
      "         1.8362e-02,  3.5670e-02, -8.8660e-03,  2.7528e-03, -2.7115e-02,\n",
      "         4.3999e-02, -6.1383e-02,  4.9109e-04, -1.3266e-02, -5.0471e-02,\n",
      "         9.9057e-03, -8.5165e-03,  2.6901e-02,  5.4465e-02, -1.5420e-02,\n",
      "         1.6378e-02, -3.8045e-02,  2.4513e-02,  8.8554e-03,  5.7986e-02,\n",
      "         1.4472e-02,  1.4161e-02,  2.9782e-02,  1.8012e-02, -1.1016e-02,\n",
      "        -1.8355e-02,  1.3665e-02, -1.3874e-02, -1.1941e-02,  4.5094e-03,\n",
      "        -4.8216e-02, -2.8180e-02, -2.1370e-03, -6.7879e-03, -9.4847e-03,\n",
      "        -1.5206e-02, -7.1828e-03,  3.7082e-02, -1.2983e-02, -1.1948e-03,\n",
      "        -2.5634e-03, -7.3714e-03,  3.2517e-02, -1.5986e-02, -1.3690e-02,\n",
      "         2.1717e-02,  5.0213e-03,  7.9712e-03,  1.5852e-02,  1.6908e-02,\n",
      "        -4.8199e-03, -1.4256e-02,  7.9553e-03,  2.6225e-03, -2.5847e-02,\n",
      "         3.2952e-03,  3.9898e-02,  6.2837e-04, -1.4863e-02,  9.3975e-03,\n",
      "         3.2082e-02,  1.3534e-02, -2.3154e-02, -1.5058e-02,  2.1762e-02,\n",
      "        -1.8215e-02, -1.1172e-02, -9.8802e-03, -2.4089e-02, -3.2108e-02,\n",
      "         3.8560e-03, -2.6522e-02, -2.2719e-02,  3.4292e-02,  3.2622e-02,\n",
      "         9.8359e-03, -3.1055e-02, -1.3631e-02,  9.8604e-04,  2.2180e-02,\n",
      "        -1.0378e-04, -8.1758e-03,  1.0198e-02, -1.3121e-03])), ('bert.encoder.layer.5.attention.output.dense.weight', tensor([[ 0.0501, -0.0722,  0.0227,  ...,  0.0413, -0.0243,  0.0188],\n",
      "        [-0.0082, -0.1109,  0.0068,  ..., -0.0362,  0.0315, -0.0287],\n",
      "        [ 0.0718, -0.0501,  0.0547,  ..., -0.0410,  0.0321,  0.0249],\n",
      "        ...,\n",
      "        [-0.0275, -0.0286,  0.0631,  ...,  0.0030,  0.0689, -0.0293],\n",
      "        [ 0.0410,  0.0952,  0.0663,  ...,  0.0275,  0.0652,  0.0109],\n",
      "        [ 0.0296, -0.0421, -0.0427,  ...,  0.0088, -0.0094, -0.0258]])), ('bert.encoder.layer.5.attention.output.dense.bias', tensor([ 0.0544,  0.0286, -0.0106, -0.0062,  0.0438, -0.0360,  0.0384, -0.0094,\n",
      "        -0.0245,  0.0286,  0.0630, -0.0388, -0.0020, -0.0301,  0.0106, -0.0251,\n",
      "        -0.0044, -0.0306, -0.0735,  0.0270,  0.0126,  0.0150,  0.0221,  0.0085,\n",
      "         0.0565, -0.0232,  0.0825,  0.0394,  0.0205, -0.0097,  0.0035, -0.0171,\n",
      "        -0.0053, -0.0045, -0.0080,  0.0165,  0.0131,  0.0079, -0.0152,  0.0048,\n",
      "         0.0587,  0.0547,  0.0165, -0.0172, -0.0153,  0.0351, -0.0036,  0.0417,\n",
      "        -0.0533,  0.0346,  0.0042, -0.0096,  0.0317,  0.0128, -0.0197,  0.0029,\n",
      "         0.0089,  0.0296, -0.0257, -0.0518,  0.0481, -0.0100, -0.0369, -0.0094,\n",
      "        -0.0301,  0.0360, -0.0156,  0.0264, -0.0141, -0.0205,  0.0456,  0.0803,\n",
      "         0.0096, -0.0270, -0.0281, -0.0325,  0.0569, -0.0301, -0.0220, -0.0639,\n",
      "        -0.0397, -0.0049,  0.0384, -0.0284,  0.0093, -0.0104, -0.0366,  0.0735,\n",
      "         0.0258,  0.0005,  0.0169, -0.0486, -0.0026,  0.0180,  0.0210, -0.0351,\n",
      "         0.0184,  0.0171, -0.0044,  0.0005,  0.0042,  0.0329, -0.0422, -0.0145,\n",
      "        -0.0162, -0.0427, -0.0089,  0.0236,  0.0366,  0.0463, -0.0339, -0.0371,\n",
      "        -0.0671,  0.0339,  0.0430,  0.0449,  0.0387, -0.0320,  0.0573, -0.0309,\n",
      "        -0.0027, -0.0172,  0.0478,  0.0209, -0.0057, -0.0454,  0.0750, -0.0341,\n",
      "         0.0334, -0.0143, -0.0023, -0.0688,  0.0279,  0.0289,  0.0447,  0.0186,\n",
      "         0.0383, -0.0151, -0.0122, -0.0646,  0.0012, -0.0117, -0.0151, -0.0009,\n",
      "         0.0087,  0.0076,  0.0231,  0.0153, -0.0449,  0.0262,  0.0240,  0.0642,\n",
      "         0.0202,  0.0371,  0.0047, -0.0213,  0.0317,  0.0337, -0.0141,  0.0113,\n",
      "        -0.0317,  0.0134,  0.0178,  0.0154, -0.0397, -0.0283,  0.0140,  0.0608,\n",
      "        -0.0203, -0.0268, -0.0097, -0.0506,  0.0101, -0.0491,  0.0117, -0.0285,\n",
      "        -0.0279, -0.0032,  0.0578,  0.0483, -0.0189, -0.0014, -0.0330, -0.0586,\n",
      "        -0.0159, -0.0254, -0.0122,  0.1144, -0.0178,  0.0271, -0.0532, -0.0073,\n",
      "         0.0095,  0.0049, -0.0346,  0.0258,  0.0722,  0.0197, -0.0230,  0.0218,\n",
      "         0.0161, -0.0189,  0.0643,  0.0015, -0.0037, -0.0015,  0.0470, -0.0136,\n",
      "        -0.0862,  0.0497, -0.0074, -0.0152, -0.0537,  0.0067, -0.0396,  0.0214,\n",
      "        -0.0012, -0.0588,  0.0363,  0.0355, -0.0849, -0.0429, -0.0405, -0.0138,\n",
      "        -0.0344,  0.0306,  0.0223,  0.0091,  0.0098,  0.0480,  0.0551, -0.0168,\n",
      "        -0.0557, -0.0235,  0.0388, -0.0442,  0.0376, -0.0007, -0.0296, -0.0020,\n",
      "        -0.0065,  0.0229,  0.0278, -0.0124, -0.0087,  0.0029, -0.0285, -0.0067,\n",
      "         0.0172,  0.0069,  0.0002, -0.0119,  0.0088, -0.0113, -0.0337,  0.0182,\n",
      "         0.0409,  0.0037, -0.0264, -0.0625,  0.0090, -0.0652, -0.0175,  0.0294,\n",
      "        -0.0037, -0.0072, -0.0395,  0.0227,  0.0543,  0.0065, -0.0629,  0.0939,\n",
      "         0.0201,  0.0003, -0.0492, -0.0104, -0.0775,  0.0356, -0.0653, -0.0126,\n",
      "        -0.0164, -0.0412, -0.0036, -0.0298,  0.0326, -0.0584, -0.0063,  0.0156,\n",
      "        -0.0337, -0.0853,  0.0345,  0.0179, -0.0343, -0.0343,  0.0052, -0.0245,\n",
      "        -0.0204,  0.0038, -0.0371,  0.0205, -0.0464, -0.0254,  0.0517,  0.0740,\n",
      "         0.0054, -0.0106, -0.0059,  0.0010,  0.0065, -0.0033,  0.0546, -0.0012,\n",
      "        -0.0518,  0.0095,  0.0102,  0.0353,  0.0107,  0.0780,  0.0265,  0.0077,\n",
      "        -0.0420,  0.0230,  0.0233,  0.0306, -0.0116,  0.0039, -0.0163, -0.0195,\n",
      "         0.0286,  0.0208, -0.0097, -0.0400,  0.0073,  0.0326, -0.0377,  0.0001,\n",
      "        -0.0480,  0.0065, -0.0418, -0.0068,  0.0589, -0.0201,  0.0638, -0.0188,\n",
      "        -0.0234, -0.0396,  0.0189, -0.0285, -0.0448,  0.0052, -0.0090,  0.0379,\n",
      "        -0.0215,  0.0006,  0.0145,  0.0662, -0.0180,  0.0701, -0.0132,  0.0299,\n",
      "        -0.0062, -0.0230,  0.0285, -0.0063,  0.0116,  0.0261, -0.0371, -0.0013,\n",
      "        -0.0207,  0.0152,  0.0076, -0.0350,  0.0098, -0.0383,  0.0288,  0.0342,\n",
      "         0.0665, -0.0627,  0.0435,  0.0443,  0.0192, -0.0666,  0.0441,  0.0186])), ('bert.encoder.layer.5.attention.output.LayerNorm.weight', tensor([0.8391, 0.8749, 0.8691, 0.8640, 0.8367, 0.8786, 0.8924, 0.8599, 0.8767,\n",
      "        0.8184, 0.9406, 0.8001, 0.8900, 0.8960, 0.8943, 0.8716, 0.8625, 0.8709,\n",
      "        0.8322, 0.8751, 0.8498, 0.8529, 0.8563, 0.8984, 0.8606, 0.8693, 0.8580,\n",
      "        0.8884, 0.8279, 0.8731, 0.8493, 1.0276, 0.8516, 0.7655, 0.8847, 0.9204,\n",
      "        0.8994, 0.8199, 0.8392, 0.9050, 0.8509, 0.8345, 0.8319, 0.8534, 0.8495,\n",
      "        0.8491, 0.8433, 0.8541, 0.8464, 0.8663, 0.8741, 0.8207, 0.8697, 0.8382,\n",
      "        0.8833, 0.8191, 0.8521, 0.8599, 0.8911, 0.9331, 0.8569, 0.8414, 0.8219,\n",
      "        0.7915, 0.8100, 0.8439, 0.8024, 0.8209, 0.8555, 0.8706, 0.8149, 0.8458,\n",
      "        0.8563, 0.8284, 1.0076, 0.8557, 0.8758, 0.8335, 0.8524, 0.8464, 0.9625,\n",
      "        0.8545, 0.9346, 0.8436, 0.8788, 0.8783, 0.8881, 0.8694, 0.8358, 0.8990,\n",
      "        0.8737, 0.9021, 0.8660, 0.8878, 0.8515, 0.8666, 0.8570, 0.8623, 0.8585,\n",
      "        0.8817, 0.8313, 0.8803, 0.8562, 0.9057, 0.9669, 0.8916, 0.8650, 0.8831,\n",
      "        0.8427, 0.9219, 0.8733, 0.8591, 0.8913, 0.8445, 0.8405, 0.8567, 0.8392,\n",
      "        0.9132, 0.8769, 0.8648, 1.1439, 0.8594, 0.8465, 0.8626, 0.8600, 0.8946,\n",
      "        0.8816, 0.9263, 0.8451, 0.9026, 0.9017, 0.9181, 0.8497, 0.8488, 0.8553,\n",
      "        0.8619, 0.8835, 0.8931, 0.8582, 0.8905, 0.8798, 0.8527, 0.8759, 0.8740,\n",
      "        0.8399, 0.8533, 0.8612, 0.8390, 0.8789, 0.8776, 0.8000, 0.9133, 0.8606,\n",
      "        0.8602, 0.8574, 0.8556, 0.8592, 0.8543, 0.8842, 0.8611, 0.8814, 0.8317,\n",
      "        0.8207, 0.8885, 0.8657, 0.8686, 0.8712, 0.8662, 0.8612, 0.8465, 0.8401,\n",
      "        0.9029, 0.8794, 0.8537, 0.8539, 0.8673, 0.8297, 0.8453, 0.7990, 0.9131,\n",
      "        0.8469, 0.8635, 0.8804, 0.8664, 0.9318, 0.8498, 0.8691, 0.4684, 0.8534,\n",
      "        0.8903, 0.9050, 0.8544, 0.8711, 0.8528, 0.9073, 0.8398, 0.8762, 0.8684,\n",
      "        0.8953, 0.8720, 0.8669, 0.8697, 0.8934, 0.8619, 0.8484, 0.8368, 0.8423,\n",
      "        0.8924, 0.9747, 0.8631, 0.8668, 0.8475, 0.8448, 0.8743, 0.8470, 0.8718,\n",
      "        0.8441, 0.8532, 0.8428, 0.8384, 0.9012, 0.9188, 0.9299, 0.8849, 0.9367,\n",
      "        0.8521, 0.8888, 0.8722, 0.8668, 0.8559, 0.8411, 0.8730, 0.8333, 0.8848,\n",
      "        0.8958, 0.8655, 0.8476, 0.8897, 0.8674, 0.8227, 0.8373, 0.8611, 0.8559,\n",
      "        0.8647, 0.8510, 0.9018, 0.8674, 0.8738, 0.6868, 0.8579, 0.8900, 0.8556,\n",
      "        0.8450, 0.9673, 0.8537, 0.8848, 0.8578, 0.8848, 0.9415, 0.8605, 0.8234,\n",
      "        0.9150, 0.8476, 0.8462, 0.9072, 0.8877, 0.8862, 0.8420, 0.8659, 0.9142,\n",
      "        0.8813, 0.9803, 0.8978, 0.8760, 0.8470, 0.9497, 0.8403, 0.9022, 0.8866,\n",
      "        0.9184, 0.8627, 0.8401, 0.8458, 0.8148, 0.8215, 0.8656, 0.8747, 0.8766,\n",
      "        0.8641, 0.8741, 0.8889, 0.8723, 0.8606, 0.9148, 0.8422, 0.8698, 0.8572,\n",
      "        0.8396, 0.9158, 0.9229, 1.0188, 0.8750, 0.8613, 0.8208, 0.8730, 0.8693,\n",
      "        0.8281, 0.8517, 0.8475, 0.8777, 0.9072, 0.8668, 0.9048, 0.8454, 0.8933,\n",
      "        0.8574, 0.8502, 0.9572, 0.8769, 0.8319, 0.8687, 0.8897, 0.8667, 0.8792,\n",
      "        0.8679, 0.8250, 0.8693, 0.8754, 0.8408, 0.8173, 0.8672, 0.8749, 0.8523,\n",
      "        0.9794, 0.8814, 0.9454, 0.9205, 0.8607, 0.9088, 0.8659, 0.8290, 0.8434,\n",
      "        0.8583, 0.8719, 0.8604, 0.8569, 0.8252, 0.9140, 0.9564, 0.8723, 0.9203,\n",
      "        0.8888, 0.8663, 0.8370, 0.8892, 0.8692, 0.9014, 0.8931, 0.8072, 0.8615,\n",
      "        0.8782, 0.8798, 0.8931, 0.8816, 0.9725, 0.9195, 0.8956, 0.8813, 0.8924,\n",
      "        0.8625, 0.8851, 0.8715, 0.8506, 0.8819, 0.8588, 0.8584, 0.8769, 0.8827,\n",
      "        0.8439, 0.8888, 0.8801, 0.9132, 0.8818, 0.8975])), ('bert.encoder.layer.5.attention.output.LayerNorm.bias', tensor([ 0.0300,  0.1011, -0.0205,  0.0463,  0.1112, -0.1326,  0.0844, -0.0475,\n",
      "        -0.1653, -0.0006,  0.1898,  0.0588, -0.1789, -0.1414, -0.2312,  0.0709,\n",
      "         0.1913, -0.0134, -0.0160, -0.0753,  0.0963,  0.1334, -0.0787,  0.2192,\n",
      "         0.1112, -0.0162,  0.0965,  0.0808, -0.0006, -0.0512, -0.1487,  0.1034,\n",
      "        -0.0722, -0.1589, -0.1552,  0.2404,  0.1618,  0.0341, -0.0715,  0.0324,\n",
      "         0.0249,  0.1703,  0.0204, -0.0284,  0.0255,  0.1492,  0.0208,  0.1600,\n",
      "         0.0377,  0.1103,  0.2714,  0.0775,  0.1465,  0.1284, -0.1612,  0.1138,\n",
      "        -0.0528,  0.0886, -0.1584, -0.2133,  0.0631,  0.0595, -0.1816,  0.0252,\n",
      "         0.0591,  0.0115,  0.0052,  0.1232,  0.0350, -0.0750,  0.0299,  0.1923,\n",
      "         0.0908,  0.0501, -0.2865,  0.0741,  0.2144, -0.0218, -0.0776, -0.2393,\n",
      "         0.0451,  0.0005,  0.2409, -0.1322,  0.0405, -0.0580, -0.1948,  0.2344,\n",
      "         0.0245,  0.2059,  0.0186, -0.2483, -0.1569,  0.1130,  0.1074,  0.1118,\n",
      "         0.0494,  0.1838, -0.1058,  0.0102, -0.0400, -0.0157,  0.0074,  0.1400,\n",
      "        -0.2648, -0.1748, -0.0139,  0.1797, -0.1789,  0.2535,  0.1034,  0.1117,\n",
      "        -0.2633, -0.0342,  0.0619,  0.2536, -0.0090, -0.1611,  0.1458, -0.0398,\n",
      "         0.2496, -0.0033,  0.1092,  0.1352,  0.0169, -0.0330,  0.0873, -0.3526,\n",
      "         0.1034,  0.1503,  0.0037,  0.1868,  0.0323, -0.0678,  0.0199,  0.0744,\n",
      "         0.1700, -0.1884, -0.0630, -0.2256, -0.1161, -0.0358, -0.1308, -0.0918,\n",
      "         0.0438,  0.0843,  0.0137,  0.0037, -0.1485,  0.0406,  0.0919,  0.2247,\n",
      "        -0.0021, -0.0687,  0.0952, -0.0339,  0.1411,  0.1766,  0.0034,  0.0439,\n",
      "         0.0461, -0.0516, -0.0243,  0.1345, -0.0372, -0.0382, -0.0060, -0.0561,\n",
      "        -0.0608,  0.0665, -0.0129, -0.3118, -0.0252,  0.1595, -0.0700,  0.1062,\n",
      "        -0.0958,  0.1239, -0.0505,  0.2636, -0.0185,  0.0332, -0.0740,  0.0418,\n",
      "        -0.1955, -0.1968,  0.0966, -0.1801,  0.0492,  0.0949,  0.0319,  0.0728,\n",
      "        -0.1331, -0.1308, -0.2566,  0.2210,  0.0865,  0.0790, -0.0856, -0.0619,\n",
      "        -0.0891, -0.0997,  0.2106,  0.2238, -0.1957,  0.0769,  0.1416,  0.0761,\n",
      "        -0.2838,  0.1473, -0.2125,  0.0310,  0.0151,  0.1141,  0.0656,  0.2576,\n",
      "        -0.0692, -0.0559, -0.1187,  0.0854, -0.2402, -0.0497, -0.2628, -0.0326,\n",
      "        -0.1867, -0.1332,  0.1113,  0.1218,  0.0198,  0.0804,  0.0248, -0.1500,\n",
      "        -0.1104, -0.0271,  0.1684, -0.0651, -0.0150,  0.0226,  0.0310,  0.0343,\n",
      "        -0.0999,  0.2848,  0.0679,  0.0646,  0.1213,  0.1772, -0.1095,  0.0981,\n",
      "        -0.1946,  0.0136,  0.0918,  0.0369, -0.0008, -0.1519, -0.0618,  0.2146,\n",
      "         0.1667, -0.1927, -0.2995, -0.1180, -0.0795, -0.3116, -0.0450, -0.0546,\n",
      "         0.1646,  0.1875,  0.0655, -0.0343,  0.0677,  0.0288, -0.1855,  0.3657,\n",
      "        -0.0759,  0.1159,  0.0011,  0.3611, -0.0630,  0.2351, -0.1262,  0.3141,\n",
      "        -0.1313, -0.0250,  0.0310,  0.0747,  0.0516, -0.0712, -0.1349,  0.1067,\n",
      "        -0.0982, -0.0669, -0.0980,  0.1624, -0.0833, -0.2080,  0.0966,  0.0937,\n",
      "         0.1261,  0.0602, -0.2278,  0.2151, -0.0964, -0.1384,  0.1947,  0.0037,\n",
      "        -0.0497,  0.1517, -0.0445,  0.0497, -0.0155,  0.0290, -0.1835,  0.1785,\n",
      "        -0.0952, -0.0303, -0.0624,  0.1542, -0.1409,  0.2752,  0.2089, -0.0851,\n",
      "         0.0952, -0.0061,  0.1308,  0.1372, -0.0299,  0.0797, -0.2029, -0.1368,\n",
      "        -0.0335, -0.0139, -0.1492, -0.1594, -0.0349,  0.2701, -0.0385,  0.2447,\n",
      "        -0.2247,  0.1429,  0.0648,  0.1522,  0.0266,  0.1093,  0.1816,  0.2107,\n",
      "        -0.2488, -0.1201,  0.0405, -0.3619, -0.2100,  0.1985, -0.2762,  0.3280,\n",
      "        -0.1210, -0.1432, -0.2105,  0.1903, -0.2444,  0.1761, -0.1497,  0.1987,\n",
      "        -0.1564, -0.0098,  0.1175,  0.1771,  0.0073, -0.1862,  0.1071, -0.1051,\n",
      "        -0.3254, -0.0528, -0.1232, -0.2371,  0.0093, -0.1854,  0.2069,  0.1109,\n",
      "         0.1208, -0.2852, -0.1527,  0.1961,  0.1007, -0.0949, -0.2134,  0.0244])), ('bert.encoder.layer.5.intermediate.dense.weight', tensor([[ 0.0288, -0.0189, -0.0309,  ...,  0.1341, -0.0092, -0.0350],\n",
      "        [ 0.0168,  0.0412,  0.0423,  ...,  0.0636,  0.0717,  0.0535],\n",
      "        [-0.0383, -0.0192,  0.0348,  ..., -0.0473, -0.0542,  0.1049],\n",
      "        ...,\n",
      "        [-0.0490,  0.0390,  0.0187,  ..., -0.0256,  0.0460,  0.0163],\n",
      "        [ 0.0849, -0.0669,  0.0120,  ..., -0.0292,  0.0239, -0.0121],\n",
      "        [-0.0023, -0.0002, -0.0187,  ...,  0.1447,  0.1044, -0.0398]])), ('bert.encoder.layer.5.intermediate.dense.bias', tensor([-0.0672, -0.0541, -0.0573,  ..., -0.0624, -0.0391, -0.0604])), ('bert.encoder.layer.5.output.dense.weight', tensor([[ 0.1365,  0.0768,  0.0380,  ..., -0.0387,  0.0493, -0.0005],\n",
      "        [-0.0717, -0.0140,  0.0194,  ..., -0.0456,  0.0855, -0.1322],\n",
      "        [ 0.1052, -0.0046,  0.0864,  ..., -0.0072, -0.0227, -0.0103],\n",
      "        ...,\n",
      "        [ 0.0842,  0.0342,  0.0115,  ..., -0.0260, -0.0615, -0.0512],\n",
      "        [-0.0021, -0.0306,  0.0511,  ..., -0.0489,  0.0111, -0.1177],\n",
      "        [-0.0359, -0.0178,  0.0023,  ..., -0.0456,  0.0431, -0.0258]])), ('bert.encoder.layer.5.output.dense.bias', tensor([-3.9028e-02, -4.1260e-02, -5.1259e-02, -1.2300e-02, -2.3105e-02,\n",
      "        -2.7247e-03, -9.1456e-03,  7.2296e-03,  1.0224e-02, -5.6274e-02,\n",
      "         5.4256e-03, -5.1449e-03, -2.7424e-02, -1.2511e-02, -7.8996e-03,\n",
      "        -5.9118e-05,  2.5663e-02,  3.0275e-02,  9.4562e-03, -6.7686e-03,\n",
      "        -3.3811e-02,  3.7768e-02, -3.1205e-04,  1.9520e-02,  6.0654e-02,\n",
      "         1.7812e-02,  1.8635e-02, -1.4182e-02,  2.8986e-02, -3.2802e-02,\n",
      "        -3.2188e-03,  2.4674e-02, -4.7878e-02, -1.5520e-02, -4.6511e-03,\n",
      "         1.3666e-02,  3.8654e-02,  1.6724e-02,  1.5690e-02,  2.0518e-02,\n",
      "        -1.9202e-02,  1.8455e-02,  1.6193e-02, -4.3098e-02, -1.6868e-02,\n",
      "        -2.5915e-02,  7.0445e-02,  5.6069e-02, -1.7251e-03, -9.3037e-03,\n",
      "         1.1061e-01, -5.2397e-02,  6.9650e-02,  1.5346e-02, -6.4926e-03,\n",
      "         4.5649e-02, -2.6783e-02,  3.7607e-02, -5.6904e-02, -4.2667e-02,\n",
      "         2.2693e-03,  8.0776e-03,  8.1021e-03,  1.9874e-02,  4.8230e-02,\n",
      "         1.6975e-03,  1.3328e-03,  7.2883e-03, -1.6668e-02,  1.2034e-02,\n",
      "         3.1274e-02,  6.3985e-02,  2.2632e-02,  1.3664e-02,  1.2118e-02,\n",
      "         6.6888e-03,  3.1653e-02,  1.2869e-02,  1.4614e-02, -3.9651e-02,\n",
      "        -5.2356e-02,  2.0935e-03,  2.4873e-02, -3.4660e-02, -6.6283e-03,\n",
      "         7.2595e-02,  3.0787e-02,  2.0323e-02, -4.2039e-02,  3.4518e-02,\n",
      "         3.9296e-02, -3.9926e-02, -7.3358e-02, -2.2788e-02, -1.4664e-02,\n",
      "         2.1662e-02, -4.2690e-02,  3.3272e-02,  8.1583e-04,  1.1079e-02,\n",
      "        -1.8837e-02,  1.8776e-02, -1.3821e-02,  3.0705e-02, -2.5177e-03,\n",
      "         1.3466e-02,  4.6722e-02,  1.9501e-02, -3.6050e-02, -8.0603e-03,\n",
      "         5.4699e-02,  1.2506e-04, -8.2124e-02,  1.4786e-02, -9.0975e-03,\n",
      "        -6.1242e-03, -5.6732e-02, -3.9010e-02,  1.6887e-02, -1.0419e-02,\n",
      "        -5.3466e-02, -1.6703e-02, -1.5027e-02,  6.0959e-04,  5.5191e-02,\n",
      "         9.9047e-03, -1.6618e-02, -1.6470e-02,  5.4560e-03,  1.0461e-02,\n",
      "        -4.8722e-02,  3.1514e-02, -2.3987e-02, -5.6919e-03, -5.4699e-03,\n",
      "        -8.9039e-03,  5.8914e-03, -7.9345e-03, -2.8713e-02,  1.2127e-02,\n",
      "        -1.4975e-02, -2.3155e-02, -2.9735e-02,  1.3280e-02,  3.1633e-02,\n",
      "         2.2647e-02,  3.4041e-02,  5.5412e-04, -3.1206e-02,  2.0659e-02,\n",
      "        -1.0411e-02,  3.4756e-02, -5.3076e-02, -3.2835e-02, -3.2593e-02,\n",
      "         8.3083e-04,  1.2418e-02,  1.9497e-02, -2.6114e-02, -4.1330e-02,\n",
      "         1.2718e-02, -2.0220e-02,  3.1186e-02, -2.2101e-02, -7.6542e-03,\n",
      "        -2.1125e-03,  4.3995e-02, -2.9362e-02, -1.2432e-02, -1.0767e-02,\n",
      "        -1.0300e-02, -4.1159e-02, -6.7529e-03,  2.1434e-02, -2.5091e-02,\n",
      "         8.9489e-04,  2.3671e-02,  6.2605e-02, -2.0149e-02,  2.9861e-02,\n",
      "         7.7849e-03,  3.5850e-02, -3.4271e-02,  3.2913e-02,  2.8489e-02,\n",
      "        -2.6488e-02,  2.8031e-02,  1.6413e-02,  2.5626e-02, -8.2858e-03,\n",
      "         3.5956e-02, -2.4047e-02, -8.4119e-02, -7.0329e-03, -8.7386e-02,\n",
      "        -1.3796e-02,  1.5467e-02,  2.1141e-03, -6.8263e-02, -9.4006e-02,\n",
      "         3.1047e-02,  3.5414e-02,  1.9727e-02,  3.1214e-02, -7.8524e-02,\n",
      "         2.4455e-02,  1.2847e-03,  1.3987e-02, -3.5818e-02,  5.2135e-02,\n",
      "         4.2982e-02,  3.3119e-03,  1.4302e-02,  6.5491e-02,  1.2479e-02,\n",
      "         2.6303e-02, -5.5721e-02, -1.8878e-02, -3.7500e-02, -1.8569e-02,\n",
      "        -7.1319e-02, -3.4777e-02, -1.8757e-02, -1.4142e-02, -8.3585e-03,\n",
      "        -4.8077e-02,  6.0065e-02,  7.4068e-03,  3.6641e-02,  1.6042e-02,\n",
      "         4.6228e-02, -4.0215e-03, -4.1626e-02, -3.2326e-03,  5.0986e-02,\n",
      "         2.6592e-02,  1.5229e-02, -2.8328e-02,  5.2046e-02,  2.1876e-02,\n",
      "         2.6753e-02,  7.4505e-02,  5.7369e-02, -7.2947e-04,  2.5622e-02,\n",
      "         2.7728e-02,  1.5478e-02, -3.6709e-02, -2.4601e-04, -2.7201e-02,\n",
      "        -2.0591e-02, -4.1181e-03,  5.6718e-02,  3.0357e-02,  2.6092e-03,\n",
      "         1.0989e-02,  5.8652e-02,  2.6481e-02, -8.0435e-02, -1.9852e-02,\n",
      "         1.9388e-02, -1.8258e-02,  2.1954e-02, -4.6043e-02,  2.4750e-02,\n",
      "         5.2044e-03, -1.1328e-02, -5.1003e-02,  6.3081e-03, -1.5068e-02,\n",
      "        -1.7594e-02, -2.3594e-03, -3.9383e-02,  5.8031e-03,  4.0999e-02,\n",
      "         6.0709e-02,  6.0121e-03, -1.1839e-02,  3.6883e-03, -1.6586e-02,\n",
      "         1.0438e-02,  1.7222e-02,  4.4173e-02, -5.6199e-02, -1.2105e-02,\n",
      "         8.3224e-03, -1.8339e-02,  4.8975e-02, -1.0859e-02, -5.3449e-02,\n",
      "        -3.1922e-02,  2.0425e-03,  2.7732e-02, -2.5895e-02,  4.6952e-02,\n",
      "        -2.9640e-03,  2.2147e-02, -2.8927e-02, -7.2602e-02,  3.2233e-03,\n",
      "        -8.7431e-03, -6.8602e-02,  5.5653e-02, -3.5773e-03, -2.7568e-02,\n",
      "         1.4601e-02, -1.0717e-02, -2.0359e-03, -1.7610e-02, -4.1205e-03,\n",
      "         3.9419e-04,  1.5658e-02,  4.7809e-03, -5.9733e-02, -6.2921e-02,\n",
      "         2.0865e-02,  5.3645e-02,  1.1661e-02,  1.3651e-02, -1.1744e-02,\n",
      "         5.2807e-02,  4.0567e-02,  9.5232e-03,  7.3301e-02,  2.1757e-02,\n",
      "         2.8098e-02, -2.9619e-02, -3.4728e-03, -2.5971e-03, -1.0727e-02,\n",
      "        -5.4553e-02, -6.7029e-03, -2.7915e-02, -4.6904e-03, -3.4580e-02,\n",
      "         3.5587e-02, -1.6758e-02,  6.2967e-02, -1.6091e-02,  3.5417e-02,\n",
      "        -1.9280e-02,  4.9208e-02,  7.7684e-02,  7.9471e-03, -3.7129e-02,\n",
      "        -1.0118e-02,  5.0855e-02, -1.7213e-02, -2.2427e-02,  2.6878e-02,\n",
      "        -5.2836e-02, -2.9585e-02,  8.9255e-03, -2.2126e-02, -3.1536e-02,\n",
      "        -5.4418e-03, -3.5187e-02, -6.6923e-03, -3.5879e-02,  4.2039e-02,\n",
      "         3.5377e-03,  2.0866e-04,  5.9162e-02, -2.0408e-02, -2.1558e-02,\n",
      "        -2.2132e-02,  1.5091e-02, -2.7907e-02, -5.4918e-02, -2.9991e-03,\n",
      "        -3.5627e-02, -2.4192e-02,  1.1520e-02, -1.3270e-02,  3.8591e-02,\n",
      "        -2.7181e-02, -2.1195e-03, -2.4588e-02, -2.7399e-02, -4.4799e-03,\n",
      "        -1.4667e-02, -4.2048e-02, -6.0997e-02,  4.2539e-02])), ('bert.encoder.layer.5.output.LayerNorm.weight', tensor([0.8561, 0.8602, 0.8797, 0.9066, 0.8809, 0.9052, 0.8953, 0.8802, 0.9042,\n",
      "        0.8713, 0.9275, 0.8529, 0.8958, 0.9302, 0.8661, 0.8768, 0.8426, 0.8910,\n",
      "        0.8624, 0.8953, 0.8847, 0.8632, 0.8662, 0.9271, 0.9008, 0.8375, 0.8701,\n",
      "        0.8899, 0.8511, 0.8741, 0.9057, 0.9248, 0.8997, 0.9117, 0.9172, 0.9026,\n",
      "        0.8577, 0.8562, 0.8198, 0.9027, 0.8889, 0.8749, 0.8509, 0.8892, 0.8698,\n",
      "        0.8866, 0.8618, 0.8743, 0.8446, 0.8932, 0.9174, 0.8874, 0.9002, 0.8204,\n",
      "        0.9412, 0.8464, 0.8736, 0.8974, 0.9133, 0.9269, 0.8662, 0.8714, 0.8683,\n",
      "        0.8331, 0.8582, 0.8890, 0.8708, 0.8684, 0.8818, 0.8863, 0.8746, 0.8858,\n",
      "        0.8624, 0.8477, 1.0162, 0.8889, 0.8878, 0.8863, 0.8483, 0.8928, 0.9357,\n",
      "        0.8676, 0.8933, 0.8556, 0.8795, 0.9224, 0.9382, 0.8606, 0.8956, 0.9153,\n",
      "        0.8860, 0.8742, 0.8717, 0.8888, 0.9065, 0.8639, 0.8738, 0.8876, 0.8631,\n",
      "        0.9040, 0.8610, 0.8882, 0.8751, 0.9059, 0.9801, 0.9066, 0.8805, 0.9004,\n",
      "        0.8936, 0.8827, 0.8718, 0.8828, 0.8824, 0.8667, 0.8862, 0.8683, 0.8904,\n",
      "        0.9133, 0.9091, 0.9005, 1.0737, 0.8691, 0.8453, 0.8834, 0.8742, 0.8784,\n",
      "        0.9106, 0.9431, 0.8709, 0.9328, 0.9066, 0.8908, 0.8698, 0.8932, 0.8696,\n",
      "        0.8649, 0.8930, 0.9062, 0.8658, 0.9101, 0.8930, 0.8598, 0.9233, 0.9157,\n",
      "        0.8777, 0.8557, 0.8658, 0.8585, 0.8594, 0.8691, 0.8667, 0.8754, 0.8909,\n",
      "        0.8850, 0.8966, 0.8784, 0.8828, 0.8781, 0.8794, 0.8799, 0.9212, 0.8854,\n",
      "        0.8669, 0.9110, 0.8861, 0.8718, 0.8996, 0.8702, 0.8803, 0.8731, 0.8716,\n",
      "        0.9080, 0.8853, 0.8773, 0.8908, 0.8726, 0.8517, 0.8829, 0.8771, 0.8834,\n",
      "        0.8914, 0.8715, 0.8661, 0.8464, 0.9527, 0.9166, 0.8836, 0.7024, 0.8761,\n",
      "        0.9101, 0.8794, 0.8962, 0.8957, 0.8722, 0.8932, 0.8660, 0.9074, 0.8944,\n",
      "        0.9016, 0.8924, 0.9064, 0.8955, 0.8944, 0.8744, 0.8823, 0.8825, 0.8745,\n",
      "        0.8774, 1.0115, 0.8388, 0.8716, 0.8848, 0.8648, 0.8598, 0.9012, 0.8736,\n",
      "        0.8812, 0.8838, 0.8665, 0.8746, 0.8604, 0.8839, 0.9040, 0.8568, 0.9080,\n",
      "        0.8578, 0.9016, 0.9101, 0.8641, 0.8882, 0.8998, 0.8906, 0.8499, 0.8553,\n",
      "        0.9091, 0.8523, 0.8724, 0.8737, 0.8777, 0.8675, 0.8569, 0.8405, 0.8891,\n",
      "        0.8661, 0.8830, 0.9289, 0.8619, 0.8940, 0.8316, 0.8755, 0.8997, 0.9262,\n",
      "        0.8970, 0.9558, 0.8504, 0.9232, 0.8987, 0.9120, 0.9009, 0.9080, 0.8561,\n",
      "        0.9058, 0.8781, 0.8705, 0.8975, 0.8874, 0.8806, 0.8645, 0.8689, 0.9196,\n",
      "        0.8901, 0.9550, 0.8978, 0.8695, 0.8914, 0.8632, 0.8903, 0.8839, 0.8487,\n",
      "        0.9030, 0.8954, 0.9121, 0.8641, 0.8220, 0.8639, 0.8921, 0.8932, 0.8699,\n",
      "        0.8851, 0.8589, 0.8856, 0.9074, 0.8901, 0.9317, 0.8657, 0.9364, 0.8472,\n",
      "        0.8825, 0.9089, 0.8950, 0.9092, 0.8707, 0.9000, 0.8705, 0.8959, 0.8880,\n",
      "        0.8959, 0.8729, 0.8773, 0.8893, 0.9047, 0.8895, 0.8927, 0.8367, 0.8828,\n",
      "        0.8818, 0.8766, 0.8831, 0.8716, 0.8903, 0.8577, 0.9013, 0.8622, 0.9150,\n",
      "        0.8692, 0.8778, 0.9022, 0.9384, 0.8633, 0.8330, 0.9222, 0.8871, 0.9058,\n",
      "        0.9221, 0.8948, 0.9336, 0.9417, 0.8677, 0.8946, 0.8858, 0.8562, 0.8603,\n",
      "        0.9050, 0.9177, 0.9100, 0.8702, 0.8789, 0.9437, 0.9080, 0.9038, 0.9327,\n",
      "        0.8659, 0.8767, 0.9026, 0.9281, 0.8914, 0.8663, 0.8915, 0.8558, 0.9341,\n",
      "        0.8973, 0.8796, 0.8719, 0.8643, 0.9255, 0.8800, 0.9041, 0.8560, 0.9327,\n",
      "        0.8481, 0.9118, 0.8846, 0.8926, 0.9024, 0.8562, 0.8752, 0.9087, 0.8342,\n",
      "        0.8446, 0.8970, 0.9190, 0.8554, 0.8753, 0.8976])), ('bert.encoder.layer.5.output.LayerNorm.bias', tensor([ 6.8790e-02,  4.3619e-03, -4.2724e-02,  2.7881e-03,  5.9462e-03,\n",
      "        -1.4101e-02,  3.0416e-02,  4.3316e-02,  6.1052e-03,  2.4260e-02,\n",
      "         3.4576e-02,  7.0948e-03, -4.8133e-02, -9.2478e-02, -6.7617e-02,\n",
      "         1.6782e-02,  9.6797e-02,  4.5979e-02,  1.5925e-02, -5.9635e-02,\n",
      "        -3.1164e-02,  2.5718e-02, -5.4915e-02,  1.2416e-01, -1.3755e-03,\n",
      "        -8.8015e-03,  3.2849e-02,  3.8934e-03, -5.3288e-02, -4.7922e-02,\n",
      "        -6.8487e-02,  3.6363e-02, -4.3768e-02, -4.9267e-02,  1.6116e-02,\n",
      "         8.9821e-02, -4.7499e-02,  5.6590e-02,  7.0090e-02, -2.0111e-02,\n",
      "        -1.0567e-01,  1.0777e-01,  9.0058e-02,  2.8673e-02,  3.0024e-02,\n",
      "        -9.3017e-03,  5.2889e-02,  8.9530e-02, -6.2895e-03,  1.7124e-04,\n",
      "         1.3375e-01,  6.9688e-02,  8.5294e-02,  4.1127e-02, -3.6056e-02,\n",
      "         6.7602e-02,  1.0316e-02, -5.6154e-02,  3.0615e-03, -6.6647e-02,\n",
      "         9.4235e-03, -2.0060e-06, -6.7054e-02, -3.0000e-02,  6.3062e-03,\n",
      "        -8.1135e-04,  6.7752e-02,  1.9840e-02, -4.3449e-02, -4.9774e-02,\n",
      "        -4.1440e-02,  1.1172e-02,  3.3409e-02, -9.8789e-02, -2.0235e-01,\n",
      "         2.7227e-02,  6.4765e-02,  3.6369e-02, -2.9449e-02, -1.0998e-01,\n",
      "         1.4334e-02, -6.3436e-02,  1.0062e-01, -3.4124e-02, -7.2253e-03,\n",
      "        -4.5208e-02, -9.4153e-02, -8.6947e-03, -7.8674e-03,  7.7398e-02,\n",
      "        -1.5548e-03, -8.3659e-02, -9.9589e-02,  6.6441e-02,  5.5666e-02,\n",
      "         8.5571e-02, -5.0802e-02,  1.6466e-01, -2.5752e-02,  4.4147e-02,\n",
      "         1.4576e-04, -1.1816e-02, -2.8052e-02,  3.5668e-02, -2.0513e-01,\n",
      "        -1.1309e-03, -2.3055e-02,  1.2041e-01, -7.0353e-02,  1.0678e-02,\n",
      "         5.5040e-02,  4.6113e-02,  6.3734e-03,  1.2054e-02,  9.2072e-02,\n",
      "         6.3838e-02, -5.6530e-02, -1.0707e-01,  3.5771e-02,  7.1740e-02,\n",
      "         1.3185e-01, -4.3907e-02,  5.7004e-02,  3.9003e-02,  1.0915e-02,\n",
      "        -7.9405e-03, -3.9611e-02, -8.1947e-02, -2.2674e-02,  8.3835e-02,\n",
      "         6.4521e-03,  1.1541e-01, -6.0582e-02, -3.1712e-02, -6.6069e-02,\n",
      "         6.3608e-02,  1.0472e-01, -7.9936e-02, -6.4117e-02, -1.3086e-01,\n",
      "        -4.6229e-02, -5.6769e-02, -5.0227e-02, -5.1529e-02,  9.3042e-02,\n",
      "         4.3237e-02, -7.2528e-02, -4.8860e-02,  1.2583e-02, -5.8763e-02,\n",
      "         5.5296e-02,  4.9033e-02, -1.3771e-02,  1.2338e-02,  8.3777e-03,\n",
      "        -2.5251e-02,  5.1590e-02,  4.1576e-02,  2.7099e-02, -6.6367e-03,\n",
      "         6.7037e-02,  5.9415e-02, -5.5165e-02,  9.1894e-02,  1.8421e-02,\n",
      "         2.4061e-02, -3.6911e-02, -5.2441e-02, -1.1863e-02,  3.9923e-03,\n",
      "        -5.6440e-02, -1.0493e-01, -5.6778e-02,  1.0932e-01, -2.4006e-03,\n",
      "         3.5786e-03,  2.4940e-02,  3.1563e-02, -3.1784e-03,  8.1995e-02,\n",
      "        -6.7126e-02,  3.5841e-02, -3.9700e-02,  7.0944e-02, -7.1945e-02,\n",
      "        -1.0883e-01,  6.8440e-02, -7.1523e-02,  4.4241e-02,  5.3922e-02,\n",
      "         1.1836e-01,  3.0296e-02, -1.1270e-01, -1.2709e-02, -1.0121e-01,\n",
      "         3.9723e-02,  1.9220e-02,  4.4339e-02,  7.1442e-02, -1.9649e-02,\n",
      "        -5.8452e-02, -3.8500e-02,  5.5810e-03,  9.5005e-02, -3.2178e-02,\n",
      "        -3.3583e-02,  8.7218e-02,  2.0709e-02, -1.5413e-01,  3.7885e-02,\n",
      "        -1.5357e-01,  3.8863e-02, -3.4543e-02, -1.1498e-02,  6.5369e-02,\n",
      "        -4.4153e-03, -4.4658e-02, -1.5419e-02, -1.6507e-02,  2.3832e-02,\n",
      "        -1.2263e-01, -2.5242e-02, -6.0381e-02,  6.7025e-02, -1.3094e-02,\n",
      "        -6.5875e-02,  4.0793e-03,  1.4289e-02,  5.6131e-02,  2.3943e-02,\n",
      "        -5.9405e-02, -6.5242e-02, -1.2363e-02, -5.8000e-02,  6.7335e-02,\n",
      "         4.3485e-02, -2.5600e-02,  2.4743e-02, -4.0804e-02,  6.3817e-02,\n",
      "        -1.5235e-02,  1.3274e-01,  1.8376e-02, -7.0193e-03,  4.5761e-02,\n",
      "         1.2292e-01, -3.2478e-02,  1.4457e-02, -3.2870e-02,  8.9862e-03,\n",
      "         8.8676e-02,  2.7918e-02, -6.9045e-03, -9.3783e-02,  6.0184e-02,\n",
      "         1.1929e-01,  8.1499e-02, -1.5293e-01, -5.3481e-02, -2.5826e-02,\n",
      "        -7.5108e-02, -9.9294e-02, -9.9606e-03, -1.0008e-01,  8.2106e-02,\n",
      "         3.3317e-02,  3.3528e-02, -4.8666e-02, -2.4153e-02,  3.7917e-02,\n",
      "        -2.6681e-02,  1.2771e-01, -3.2416e-02, -4.9995e-02,  3.2732e-02,\n",
      "         1.1974e-01,  2.9857e-02,  5.2595e-02,  2.8713e-02,  6.8816e-02,\n",
      "        -3.2203e-02, -1.0321e-02,  5.5162e-02, -2.3682e-02, -4.0805e-02,\n",
      "        -3.2745e-02, -9.8242e-03,  4.6349e-03,  5.1749e-02, -2.1660e-02,\n",
      "        -1.8000e-02,  6.6053e-02, -2.4825e-02, -1.1718e-01,  1.0909e-01,\n",
      "         1.4265e-01,  9.1980e-02,  4.6700e-02, -5.9907e-02,  2.9457e-03,\n",
      "        -3.6476e-02, -1.1038e-02,  4.9717e-02,  2.3535e-03, -3.8571e-02,\n",
      "         8.9338e-02, -2.1507e-02,  2.6675e-02, -2.7102e-02,  1.9327e-02,\n",
      "        -1.1482e-01,  1.1043e-01, -3.1307e-02, -2.4144e-02, -5.1455e-02,\n",
      "        -4.2360e-02, -1.1380e-02,  1.0771e-01,  1.1209e-01, -3.7215e-02,\n",
      "        -7.3275e-02, -1.8103e-02,  7.0301e-02,  6.5701e-02,  2.1590e-02,\n",
      "         4.1557e-02, -1.0608e-01, -8.3584e-02, -3.6066e-02, -3.7307e-02,\n",
      "        -9.8904e-02, -1.0477e-01, -4.6969e-03,  1.1105e-01, -2.1331e-02,\n",
      "         7.1341e-02, -1.1690e-01,  7.4738e-02,  1.1268e-01,  1.4036e-01,\n",
      "        -2.8874e-02,  4.3351e-02,  5.4001e-02,  8.8136e-02, -1.1762e-01,\n",
      "        -1.5391e-02, -4.5754e-02, -1.3164e-01,  2.9887e-02,  1.5436e-01,\n",
      "        -1.2572e-01,  1.0158e-01, -7.6720e-02, -1.6891e-01, -6.3570e-02,\n",
      "        -2.1308e-02, -4.2366e-02,  8.3050e-02, -1.4885e-02,  3.6043e-02,\n",
      "        -7.2238e-02, -2.4083e-02,  8.3297e-02,  3.7696e-02,  1.3318e-02,\n",
      "        -5.4267e-02,  1.0874e-01, -1.6156e-02, -1.2158e-01, -3.7059e-02,\n",
      "        -1.0452e-01, -1.6372e-01, -2.9381e-02, -8.0917e-02, -1.2069e-02,\n",
      "        -3.5111e-02,  5.5243e-02,  4.3356e-03,  1.9145e-02,  5.8595e-02,\n",
      "         6.2760e-02,  4.3490e-02, -1.7390e-01, -1.2611e-02])), ('bert.pooler.dense.weight', tensor([[ 0.0991, -0.0047, -0.0061,  ...,  0.0188, -0.0159, -0.0510],\n",
      "        [-0.0552,  0.0452,  0.0023,  ...,  0.0122, -0.0227, -0.0012],\n",
      "        [-0.0068, -0.0174, -0.0054,  ...,  0.0099, -0.0235,  0.0038],\n",
      "        ...,\n",
      "        [-0.0359,  0.0283, -0.0045,  ..., -0.0014, -0.0171,  0.0233],\n",
      "        [-0.0016, -0.0298,  0.0048,  ...,  0.0208, -0.0263, -0.0004],\n",
      "        [ 0.0216,  0.0372,  0.0281,  ..., -0.0040,  0.0092, -0.0112]])), ('bert.pooler.dense.bias', tensor([ 6.1237e-03, -7.6239e-03, -3.7915e-03,  7.8844e-03,  1.1873e-01,\n",
      "         1.7247e-02,  9.3152e-03,  1.8738e-02, -3.2643e-03, -3.9130e-03,\n",
      "         3.8753e-03, -4.1539e-02,  2.1087e-02, -9.2521e-03,  1.8342e-03,\n",
      "         2.5355e-03,  6.4768e-03, -4.3751e-03, -2.0299e-04, -7.2715e-03,\n",
      "        -8.8383e-03,  8.8377e-03, -4.9131e-04, -5.7100e-03, -7.6946e-03,\n",
      "         3.6952e-03, -8.7946e-03, -1.0183e-02, -2.9022e-03, -5.1023e-03,\n",
      "        -1.2110e-02, -1.4346e-03, -1.1063e-03, -3.8468e-03, -3.7797e-03,\n",
      "         6.9688e-03, -1.1657e-03, -7.6946e-03, -7.9466e-03,  1.0276e-03,\n",
      "        -1.5764e-02, -1.2430e-02,  5.0828e-03,  2.6943e-03, -5.9697e-03,\n",
      "         9.3199e-03, -9.6074e-03,  4.9663e-02,  2.4912e-03, -4.3754e-03,\n",
      "         5.9844e-02,  1.0348e-02,  1.0202e-01,  1.4562e-02,  1.5872e-02,\n",
      "        -5.7059e-03,  4.2009e-03,  1.0790e-03, -3.0567e-03,  7.1055e-03,\n",
      "         1.1970e-02, -1.5363e-01, -9.7009e-03, -3.2381e-03,  7.7459e-02,\n",
      "         4.4683e-03,  1.0023e-01, -1.6783e-02,  5.2460e-03,  1.5039e-01,\n",
      "         1.8815e-03,  4.3476e-03, -1.2895e-01,  1.9702e-02,  2.9866e-03,\n",
      "        -1.2726e-02, -4.4772e-02, -6.3041e-03, -3.4674e-03, -7.7993e-02,\n",
      "        -4.3272e-03, -8.0871e-03,  1.0444e-02, -3.9287e-03, -1.4862e-01,\n",
      "         1.1175e-01, -1.7605e-03,  3.7933e-03,  7.0097e-03, -1.1800e-02,\n",
      "        -2.2288e-03,  8.5687e-02,  1.2197e-02,  9.7673e-03,  6.6539e-03,\n",
      "        -6.5398e-04, -4.0523e-02,  3.4511e-02, -6.4598e-02,  4.9390e-03,\n",
      "         1.0901e-02, -9.2594e-02, -5.0585e-03, -5.6534e-02, -1.3706e-02,\n",
      "        -8.7467e-03,  5.1652e-03, -1.9562e-03,  4.8207e-03, -5.5576e-03,\n",
      "        -3.4290e-03,  1.7865e-02, -1.6582e-02, -9.8821e-02,  7.6505e-03,\n",
      "         3.4009e-02, -9.1888e-03,  1.4074e-02,  1.4647e-01, -6.1829e-02,\n",
      "        -9.2572e-03,  6.5068e-03,  3.5832e-02, -5.7623e-03,  6.4843e-03,\n",
      "         9.5864e-03,  1.9476e-02, -3.5605e-03, -1.4198e-02, -1.5287e-02,\n",
      "         1.2330e-02,  1.6267e-01,  9.4976e-04, -7.4822e-03, -7.5809e-03,\n",
      "         1.0587e-01, -1.2280e-02, -3.6982e-02, -1.6993e-02, -1.3376e-01,\n",
      "        -6.7649e-03,  1.5910e-01,  8.7126e-02, -7.8336e-03, -5.1435e-03,\n",
      "        -2.3772e-03, -3.2647e-02,  6.6948e-03,  4.2942e-03, -1.5811e-02,\n",
      "        -8.4470e-03, -1.7654e-02, -9.6604e-02, -6.6691e-02,  1.1127e-03,\n",
      "        -6.4540e-03, -6.2896e-02, -1.0907e-02, -2.3768e-02,  9.1271e-04,\n",
      "         2.3942e-03,  6.1002e-03, -1.1232e-02,  7.2155e-02, -7.8690e-03,\n",
      "        -6.1256e-04, -1.0177e-02, -1.4970e-01, -1.2827e-02,  4.0294e-03,\n",
      "         1.1720e-02, -1.3937e-02, -2.0478e-03, -1.7505e-03,  1.1034e-02,\n",
      "         9.0915e-02, -7.0046e-03, -1.8514e-02, -9.0010e-03,  1.1770e-02,\n",
      "         9.8294e-03,  4.6535e-03,  6.1319e-03,  1.0406e-02, -1.1585e-01,\n",
      "        -8.7574e-03, -9.5137e-03,  1.2856e-01,  5.0191e-03, -7.6159e-02,\n",
      "        -5.9693e-02,  7.7584e-03, -5.7605e-03, -1.0387e-01,  4.2293e-03,\n",
      "        -6.8734e-03, -2.0138e-03, -9.1251e-04, -7.9403e-03, -2.8626e-03,\n",
      "        -5.1892e-03, -2.6515e-03,  4.6069e-05,  2.4061e-04,  5.8025e-03,\n",
      "        -3.9215e-03,  1.1975e-01, -3.0634e-03,  1.5832e-04, -3.5108e-03,\n",
      "         3.5241e-02, -5.0375e-02,  9.0307e-03,  1.1780e-02,  5.7008e-02,\n",
      "         5.6656e-04,  6.8526e-02,  6.5357e-03,  1.6065e-02,  9.4817e-05,\n",
      "        -6.8243e-03,  6.5474e-03, -9.7123e-03,  5.9214e-03, -1.8637e-03,\n",
      "        -1.3168e-02,  3.9010e-02, -3.8168e-03,  1.1810e-06, -5.7281e-03,\n",
      "        -4.2165e-03,  1.1092e-02, -1.2641e-03, -5.2519e-03,  5.8241e-03,\n",
      "         4.4751e-03, -2.6494e-03, -4.0418e-03, -9.7615e-02,  3.8316e-03,\n",
      "        -6.2858e-03, -2.3871e-02, -3.1591e-06, -5.0885e-03,  5.6644e-03,\n",
      "        -1.1832e-03,  1.4575e-02, -5.5560e-03,  1.8141e-02,  3.3237e-03,\n",
      "        -7.1528e-03,  1.8880e-02, -5.0650e-03,  5.5348e-03,  2.2800e-03,\n",
      "         6.5755e-02,  1.8086e-03,  3.4284e-02,  4.9714e-03,  1.0661e-02,\n",
      "        -1.3847e-01,  1.6295e-02, -3.0634e-02,  1.8783e-02,  1.2706e-02,\n",
      "         3.6925e-03, -1.4333e-03,  3.3283e-03,  1.0836e-02,  1.3722e-02,\n",
      "         8.2180e-02, -1.9860e-02,  1.8996e-02,  1.5278e-02,  3.7254e-03,\n",
      "         3.3255e-02,  3.8709e-02,  4.1354e-03, -3.8872e-03, -1.2294e-03,\n",
      "        -1.6123e-03,  1.9010e-02,  9.0617e-03, -9.4454e-03, -3.6081e-03,\n",
      "         9.1886e-02,  9.3122e-03,  1.9304e-03,  2.4478e-03,  1.2805e-02,\n",
      "        -1.0075e-02, -2.5910e-02,  1.3603e-01,  6.0832e-03, -3.3871e-04,\n",
      "         2.1388e-02,  4.6348e-02,  2.1866e-03,  8.2030e-02, -9.3878e-04,\n",
      "         1.6027e-03, -8.9188e-03, -6.4827e-03,  1.5548e-01,  4.1103e-03,\n",
      "        -2.3028e-03, -9.4589e-03,  5.8285e-02,  1.1014e-03, -6.4513e-03,\n",
      "         1.7788e-02,  2.4858e-02,  1.0558e-01,  8.5413e-02,  7.7098e-02,\n",
      "         1.1934e-01, -1.5072e-02,  7.2490e-03,  3.9723e-03, -8.4964e-05,\n",
      "         3.8979e-03, -1.3055e-03,  8.1726e-02, -8.0465e-03,  5.8247e-03,\n",
      "        -2.2551e-06, -1.1591e-03,  7.2936e-02, -9.1361e-02, -3.4826e-03,\n",
      "        -1.7552e-02,  7.8280e-03, -5.1797e-03,  1.1614e-02, -4.0980e-03,\n",
      "        -6.8905e-03, -1.2944e-02, -3.8402e-02, -7.8327e-05, -6.8673e-03,\n",
      "         1.7980e-02, -1.3238e-02, -9.4982e-02,  5.2165e-03,  3.6817e-03,\n",
      "        -9.3463e-03, -4.1533e-02, -2.1478e-02, -5.3686e-03, -1.8450e-03,\n",
      "         3.2282e-03, -1.2606e-02, -4.9744e-03,  4.9056e-03, -1.1982e-02,\n",
      "         1.5670e-02, -5.8974e-02, -9.2281e-03, -6.0041e-04,  4.9779e-03,\n",
      "        -6.1128e-03,  4.4955e-03,  1.2340e-02, -5.4941e-03,  4.0856e-02,\n",
      "         3.5459e-03, -3.1819e-02, -5.0862e-04, -1.5278e-01, -9.9460e-02,\n",
      "         4.2554e-02, -1.1831e-02,  1.2210e-01, -2.7729e-03,  3.1277e-02,\n",
      "         7.0737e-04, -8.8525e-03, -1.6551e-02, -4.5013e-03, -1.0368e-01,\n",
      "        -7.1417e-03,  1.4726e-03,  5.0964e-03, -1.8553e-02])), ('cls.predictions.bias', tensor([ -0.7120,   0.3262, -15.6331,  ...,  -3.5982,  -0.7439,  -1.0157])), ('cls.predictions.transform.dense.weight', tensor([[ 0.3037,  0.0374, -0.1312,  ..., -0.0789,  0.0567,  0.0643],\n",
      "        [-0.1054, -0.0117,  0.0260,  ..., -0.0009, -0.0603,  0.1077],\n",
      "        [-0.0132, -0.1336,  0.3336,  ..., -0.0187, -0.1191, -0.0504],\n",
      "        ...,\n",
      "        [ 0.0019, -0.0413,  0.0659,  ...,  0.4077, -0.0361, -0.0285],\n",
      "        [-0.1250,  0.0674, -0.0748,  ..., -0.0742,  0.3121, -0.0273],\n",
      "        [ 0.0549, -0.0287,  0.0355,  ..., -0.1262, -0.0688,  0.5201]])), ('cls.predictions.transform.dense.bias', tensor([ 8.8099e-02,  1.7940e-01,  1.3668e-02,  8.1220e-02,  5.6441e-02,\n",
      "        -2.8639e-03,  3.6110e-02, -4.1917e-02, -1.1919e-01,  2.0284e-03,\n",
      "        -9.7005e-02, -6.3484e-02, -3.7624e-02, -3.7368e-02, -1.0059e-01,\n",
      "        -1.7682e-02,  5.9755e-01,  6.7681e-02,  7.5590e-02, -8.1154e-03,\n",
      "         3.2743e-02, -1.2643e-02,  1.7392e-03,  1.0366e-01,  5.3330e-02,\n",
      "         5.7554e-01, -4.0414e-02,  1.5378e-01,  2.0274e-01, -4.5296e-02,\n",
      "         7.9759e-03,  7.2192e-02, -2.8322e-02,  2.1940e-02, -6.6203e-03,\n",
      "         1.4411e-01,  1.7677e-01, -6.9613e-02,  1.8253e-02,  1.2806e-01,\n",
      "        -7.4301e-02, -9.5446e-03,  8.9842e-02,  5.1782e-02, -2.0934e-02,\n",
      "        -3.1375e-02,  1.2057e-01,  1.3661e-02,  5.4776e-02,  8.0858e-02,\n",
      "         1.7766e-01,  5.6044e-03,  5.6905e-02,  1.0336e-01,  1.4192e-01,\n",
      "         2.0577e-01, -9.2626e-02, -1.2585e-02,  6.9882e-02,  6.6827e-02,\n",
      "        -5.9115e-05, -1.3747e-03,  1.6942e-01, -6.7378e-03, -8.2009e-02,\n",
      "         2.7788e-01,  2.3842e-01,  4.2592e-03, -8.5652e-02,  4.8798e-02,\n",
      "         5.0786e-02,  2.2471e-01,  1.6473e-01, -1.0527e-01, -4.7191e-02,\n",
      "         1.6584e-01,  1.0721e-01,  1.8686e-01,  1.9616e-01,  9.0763e-02,\n",
      "         9.2313e-02,  6.7962e-02,  8.1847e-02,  1.0507e-01, -4.1172e-02,\n",
      "        -2.0213e-01,  1.8859e-01, -8.2150e-02,  6.6728e-02,  3.3893e-02,\n",
      "         2.2877e-01,  1.5034e-01,  1.5634e-02, -4.4773e-02,  7.4754e-02,\n",
      "         4.9541e-02, -2.6343e-03,  3.8441e-02,  5.4651e-02,  1.6187e-01,\n",
      "         3.0550e-02, -4.7625e-02,  9.7695e-02,  9.7152e-02,  1.2067e-01,\n",
      "        -9.7765e-02,  7.3234e-02,  6.7756e-03,  9.8852e-02, -2.4953e-02,\n",
      "         1.3892e-01, -4.1569e-03,  2.5573e-02,  1.3489e-01,  1.2164e-01,\n",
      "         1.3727e-01, -4.1897e-02, -3.8998e-02, -2.9153e-03, -5.9008e-02,\n",
      "         1.0068e-01,  3.1585e-02,  1.2207e-02, -6.5526e-03,  1.8110e-01,\n",
      "         5.0941e-02,  2.2245e-02, -4.7249e-02,  1.0859e-01,  4.8323e-02,\n",
      "        -2.2070e-02,  9.7684e-02,  3.1238e-04,  3.4205e-02, -1.8970e-03,\n",
      "        -1.2024e-02,  1.0148e-01, -7.7474e-04,  8.3993e-02,  2.9024e-02,\n",
      "         1.5592e-02, -3.4831e-02,  1.2484e-01,  1.5914e-02,  6.9672e-02,\n",
      "         1.5622e-01, -4.5716e-03,  3.4569e-02,  3.7126e-02,  3.8315e-02,\n",
      "         1.3775e-01, -1.4142e-01,  2.2216e-02,  1.7985e-01, -5.1981e-02,\n",
      "        -6.8288e-02,  1.4721e-02,  7.5837e-02,  9.3965e-02,  8.1353e-03,\n",
      "         1.1872e-01,  1.4744e-01,  5.4473e-02,  1.2433e-01, -2.1977e-03,\n",
      "         1.2854e-01, -1.4058e-03,  1.9676e-01,  1.3549e-01,  7.9724e-02,\n",
      "         1.1966e-02, -1.2552e-02, -4.2748e-02, -3.6991e-02,  7.9101e-02,\n",
      "         8.1145e-03,  9.8754e-02,  1.1280e-01, -1.6551e-01, -1.2565e-03,\n",
      "         8.9373e-02,  1.9257e-01,  8.0437e-02,  1.7723e-01, -8.9055e-03,\n",
      "         1.2012e-01, -6.3901e-02,  8.0358e-02,  3.3199e-02, -8.0170e-02,\n",
      "         1.2626e-01,  6.4978e-02,  7.4128e-03,  1.4314e-01,  1.2417e-01,\n",
      "         2.3235e-01,  2.0838e-01,  1.1068e-01,  5.5286e-02,  1.2651e-02,\n",
      "         4.3520e-02,  6.1069e-02,  1.8796e-01,  1.4216e-01,  5.7022e-02,\n",
      "         9.8652e-02, -1.6667e-01, -1.3436e-01,  7.3920e-02, -2.6562e-02,\n",
      "        -1.6156e-02,  3.3224e-02,  6.8120e-02,  3.5374e-02, -3.9894e-02,\n",
      "        -3.4328e-02,  9.7568e-03, -1.1660e-01, -4.9962e-02,  7.6241e-02,\n",
      "         1.2395e-01, -8.9032e-02, -7.6266e-03,  5.5620e-03,  8.5245e-02,\n",
      "         1.3091e-01,  1.1325e-02,  1.9504e-01,  8.1247e-02, -1.6294e-02,\n",
      "         5.1097e-02,  1.0390e-01,  1.7872e-01,  7.9064e-02,  1.9451e-01,\n",
      "        -1.3503e-02, -1.1196e-01,  1.4834e-01,  5.1253e-02,  7.2526e-02,\n",
      "         1.5520e-01,  4.4641e-04,  1.0562e-01, -6.9058e-02,  2.5465e-02,\n",
      "         2.1057e-01,  2.1154e-02, -2.2661e-01,  2.0987e-01,  5.8169e-02,\n",
      "         4.6106e-02,  5.6639e-02,  6.5659e-02,  1.4140e-01,  4.4246e-02,\n",
      "        -1.5580e-02,  2.2433e-01,  1.2917e-01,  1.1602e-01,  7.0207e-02,\n",
      "         1.1399e-01,  2.3577e-02,  1.1509e-01,  9.5778e-02,  1.2835e-01,\n",
      "        -6.9480e-02,  1.6593e-01, -1.6398e-02, -2.2441e-02,  9.5549e-03,\n",
      "         9.4674e-02, -1.4609e-02,  5.7250e-02,  4.8192e-03,  1.5952e-01,\n",
      "        -4.6869e-02,  1.2474e-01,  1.0000e-01, -7.3721e-02, -1.2531e-01,\n",
      "         1.0782e-01,  2.3194e-02,  3.9520e-02,  1.4020e-01,  5.2987e-02,\n",
      "         1.2113e-01,  1.0761e-03, -9.1629e-02,  9.8693e-02,  1.2787e-02,\n",
      "        -5.4796e-02, -3.2980e-02,  1.0190e-01,  1.2388e-01,  2.6604e-01,\n",
      "         6.7991e-02,  1.4611e-01,  1.7324e-01, -7.4465e-03,  7.0824e-02,\n",
      "        -9.4629e-02, -1.0072e-01,  5.9880e-02,  9.8021e-02, -2.1851e-02,\n",
      "         3.8535e-02,  7.1958e-02,  1.7550e-01,  1.7504e-01,  5.5918e-02,\n",
      "        -2.6609e-02, -2.2633e-01,  1.0496e-01, -2.8186e-02,  5.1224e-02,\n",
      "         2.4579e-02,  5.0851e-02,  9.0590e-03,  1.4582e-01,  1.1713e-01,\n",
      "         7.1458e-02,  5.6495e-02, -1.8929e-03,  4.5609e-02,  7.9304e-02,\n",
      "         6.9300e-02, -8.5355e-02, -8.4150e-03, -7.9345e-03,  1.1845e-01,\n",
      "         1.6284e-01,  2.6969e-01,  1.4936e-01, -1.5764e-01, -1.0960e-01,\n",
      "        -2.2215e-02,  1.3477e-01,  2.2599e-01,  1.7678e-01,  1.6337e-01,\n",
      "         1.3712e-01,  9.2603e-02,  8.1541e-02,  1.7937e-01, -6.5451e-02,\n",
      "         1.8494e-01, -1.6959e-02,  5.4411e-03,  2.5246e-01,  1.9497e-01,\n",
      "         1.4168e-01, -4.0259e-02,  2.0908e-01, -3.2260e-02,  3.0142e-02,\n",
      "         1.6960e-02,  4.6793e-02,  2.8839e-02,  1.1627e-01,  6.2894e-02,\n",
      "         1.4389e-01, -1.3083e-01,  1.0011e-01, -1.6441e-01,  7.3251e-02,\n",
      "        -1.3953e-01,  1.2813e-01,  3.7641e-02,  2.8498e-02,  1.6454e-01,\n",
      "         1.7196e-01, -7.6279e-02,  1.6529e-01,  5.3977e-02,  1.7416e-01,\n",
      "        -1.8255e-01,  4.0760e-02,  2.3847e-01,  1.2739e-01,  1.2001e-01,\n",
      "        -4.3749e-03,  2.2398e-02,  5.4757e-02,  1.2078e-01])), ('cls.predictions.transform.LayerNorm.weight', tensor([ 2.9070,  5.6163,  2.4316,  2.9666,  2.9873,  2.7900,  2.7063,  2.3592,\n",
      "         2.8914,  2.7557,  2.5990,  2.7203,  2.9521,  2.7768,  2.3812,  2.6655,\n",
      "         0.6711,  2.3917,  2.4618,  2.5049,  2.3139,  2.6140,  2.2875,  3.0239,\n",
      "         3.1006,  0.0622,  5.7799,  2.6199,  2.1213,  2.3443,  2.0794,  2.2759,\n",
      "         2.2537,  2.3558,  2.3362,  2.1993,  2.3271,  2.2273,  2.3466,  2.1553,\n",
      "         2.1130,  0.2080,  2.0346,  2.1501,  2.2377,  2.1010,  2.1833,  2.0300,\n",
      "         2.0520,  2.2267,  2.0745,  1.9339,  2.2555,  1.9568,  2.0749,  2.0857,\n",
      "         2.1792,  2.0890,  2.0265,  2.0968,  2.1304,  2.1474,  1.9276,  2.0409,\n",
      "         1.6720,  1.9977,  2.0505,  2.1327,  2.1412,  2.1536,  2.0624,  2.1100,\n",
      "         2.1074,  2.0548,  2.0913,  1.9891,  1.9339,  1.9837,  1.9948,  2.1437,\n",
      "         2.0158,  2.1527,  2.1344,  2.1207,  2.1506,  1.9520,  2.1047,  1.9582,\n",
      "         2.0294,  2.0933,  2.2969,  2.1093,  2.2106,  2.1200,  2.0031,  2.2548,\n",
      "         2.2967,  2.2454,  2.2148,  2.2403,  2.1968,  0.0877,  2.1994,  2.0233,\n",
      "         2.1903, -0.1380,  2.2788,  2.1033,  2.3290,  2.2725,  2.2014,  2.2333,\n",
      "         2.2815,  2.1833,  2.1314,  2.2789,  2.2376,  2.3437,  2.2698,  0.3515,\n",
      "         2.3575,  2.2788,  2.2574,  2.2745,  2.1987,  2.1753,  2.3696,  2.1014,\n",
      "         2.0863,  2.1968,  2.0888,  2.2317,  2.1967,  2.1932,  2.2667,  2.2604,\n",
      "         2.3334,  2.1906,  2.2403,  2.1717,  2.2834,  2.2673,  2.3693,  2.2250,\n",
      "         1.9737,  2.4201,  2.2391,  2.0801,  2.2481,  2.2839,  2.1149,  2.0555,\n",
      "         2.2294,  2.2625,  1.7025,  2.2506,  2.1339,  2.9519,  2.2073,  1.9775,\n",
      "         2.3176,  2.1434,  2.2312,  2.3552,  2.1933,  2.0884,  2.3099,  2.1529,\n",
      "         1.9060,  2.3742,  2.1907,  1.9817,  2.1998,  2.3030,  2.3496,  2.2722,\n",
      "         2.4704,  2.0859,  2.0079,  2.8718,  2.3161,  1.8899,  2.2716,  2.0920,\n",
      "         2.2464,  2.0726,  2.2588, -1.7840,  2.3123,  2.1243,  2.1987,  2.2914,\n",
      "         2.2336,  2.1354,  2.1267,  2.1318,  2.2468,  2.2499,  2.1605,  2.2498,\n",
      "         2.4126,  2.1290,  2.1081,  2.2756,  2.2491,  2.1096,  2.0336,  2.1085,\n",
      "         2.3134,  2.2543,  2.3265,  2.2286,  2.3620,  2.2396,  2.2043,  2.1950,\n",
      "         2.2764,  2.1205,  1.9358,  2.3707,  2.1642,  2.2790, -0.0183,  2.2874,\n",
      "         2.1310,  2.2035,  2.1763,  2.0942,  2.3745,  2.1878,  1.9760,  2.4731,\n",
      "         2.2043,  2.2821,  2.2052,  2.4092,  0.0609,  2.2269,  2.2890,  2.3429,\n",
      "         2.1047,  2.3314,  2.3434,  2.2789,  2.5525,  1.9974,  2.1462,  1.6970,\n",
      "         2.0118,  2.2615,  2.3002,  2.2376,  2.5047,  2.3540,  2.3285,  2.1252,\n",
      "         2.4276,  2.3247,  1.9203,  2.2630,  2.4581,  2.3401,  2.2525,  2.0968,\n",
      "         2.2679,  2.1907,  2.2073,  2.3129,  2.3899,  2.2528,  2.4054,  2.1893,\n",
      "         2.2914,  2.1855,  2.0621,  2.0907,  2.3247,  2.2185,  2.0698,  2.0922,\n",
      "         2.2915,  2.3393,  2.3189,  4.2543,  2.2195,  2.2699,  2.2484, -0.1549,\n",
      "         2.1348,  2.2771,  2.2868,  2.1422,  2.2479,  2.0734,  2.0397,  2.2439,\n",
      "         2.3960,  2.1570,  2.3387,  3.7511,  2.2010,  2.1648,  2.3973,  2.0689,\n",
      "         2.2614,  2.2073,  2.1693,  2.1417,  2.1646,  2.3037,  2.1948,  1.7070,\n",
      "         2.1766,  2.2040,  2.1695,  2.2173,  2.3572,  2.4136,  2.2738,  2.0562,\n",
      "         2.3353,  2.1421,  2.3633,  2.4235,  2.3002,  2.3395,  2.2093,  2.4056,\n",
      "         2.2521,  1.9714,  2.1828,  2.1711,  2.1016, -0.0137,  1.9029,  2.1415,\n",
      "         2.2780,  1.9554,  2.3558,  2.1942,  2.2511,  2.2361,  2.2290,  2.0594,\n",
      "         2.0885,  2.2208,  2.2313,  2.2102,  2.2002,  2.1431,  2.2026,  2.2295,\n",
      "         2.2822,  2.1090,  2.2634,  2.2080,  2.3142,  2.2902,  2.1923,  2.1356,\n",
      "         2.0851,  2.1617,  2.2038, -0.1964,  2.1785,  2.0615,  2.3140,  2.1689,\n",
      "         2.1612,  1.9939,  2.0854,  2.0775,  2.2393,  2.3399,  2.3918,  1.9711,\n",
      "         2.2805,  2.1353,  2.1904,  2.2279,  1.8524,  2.2238,  2.2774,  2.1755])), ('cls.predictions.transform.LayerNorm.bias', tensor([ 3.9389e-01, -2.9514e+00,  3.7916e-01,  5.0254e-01,  3.9970e-01,\n",
      "         4.0600e-01,  4.9535e-01,  3.4922e-01, -8.0434e-02,  2.9022e-01,\n",
      "         1.4377e-01, -1.9365e-02,  2.3508e-01, -2.4682e-02, -7.1165e-02,\n",
      "         1.0908e-01, -1.4429e+00, -2.6459e-02,  4.1884e-01, -1.4156e-01,\n",
      "         8.3866e-02,  1.3320e-01,  1.7211e-01,  2.8151e-01, -6.1942e-03,\n",
      "        -1.4227e+00, -1.0395e+00,  3.4552e-01,  3.6130e-01, -1.0949e-01,\n",
      "         1.0434e-02, -2.9494e-02, -1.4399e-02,  6.9492e-02, -1.0566e-02,\n",
      "         4.5684e-01,  4.5027e-01, -8.3174e-02, -4.1314e-03,  2.0975e-01,\n",
      "        -5.0025e-02,  4.9615e-01,  3.0841e-01,  7.0332e-02, -9.2594e-02,\n",
      "        -3.6030e-02,  4.2375e-01, -8.7322e-02,  2.2938e-01,  2.9543e-01,\n",
      "         4.6204e-01,  4.1833e-01,  1.5327e-01,  2.0263e-01,  3.7471e-01,\n",
      "         3.3288e-01, -2.7752e-01,  1.1667e-01, -6.0334e-02, -2.1544e-02,\n",
      "        -9.7099e-03, -5.3383e-02,  4.4007e-01, -4.3248e-02,  1.5404e-01,\n",
      "         4.7438e-01,  5.0439e-01, -2.9170e-02, -1.2834e-01, -1.1336e-02,\n",
      "         2.1731e-01,  3.8337e-01,  3.5923e-01, -3.1662e-01, -2.5422e-01,\n",
      "         3.2214e-01,  2.6757e-01,  3.8260e-01,  1.1844e-01,  1.1840e-02,\n",
      "         1.0026e-01,  9.6624e-03,  1.9811e-01, -9.4404e-02, -9.0845e-02,\n",
      "        -2.7015e-01,  4.4609e-01, -3.2710e-02, -1.2762e-02, -1.0004e-02,\n",
      "         4.2221e-01,  1.6962e-01,  8.8940e-03, -1.4045e-01,  1.5917e-01,\n",
      "         1.4318e-01, -9.6096e-02,  8.1027e-02,  1.8167e-02,  1.1023e-01,\n",
      "        -1.3842e-01,  5.6122e-01,  3.2937e-01,  2.3603e-01,  2.7025e-01,\n",
      "         7.0755e-01,  1.6763e-01, -6.5718e-02,  8.9146e-02, -1.0689e-01,\n",
      "         1.1610e-01, -1.1286e-01,  6.9010e-02,  2.7819e-01,  2.8510e-01,\n",
      "         1.7495e-01, -1.2015e-01, -1.4127e-01, -1.1878e-01,  3.8195e-01,\n",
      "         9.4102e-02, -7.7078e-03,  5.6792e-02, -8.9125e-02,  3.0933e-01,\n",
      "         2.5515e-01, -2.7506e-02, -1.7404e-01,  2.7538e-01,  1.5272e-01,\n",
      "        -9.4431e-02,  8.1885e-02,  7.8534e-02,  3.1250e-02, -1.5583e-02,\n",
      "        -4.0683e-02,  7.2507e-02, -7.0295e-02,  1.1671e-01,  1.3694e-01,\n",
      "        -6.4092e-02, -9.9194e-02, -1.0536e-02, -4.5869e-02,  1.1150e-01,\n",
      "         2.7570e-01, -1.5733e-01,  1.8094e-01, -1.6175e-02,  3.4845e-02,\n",
      "         3.6070e-01, -2.0151e-01,  2.0195e-01, -9.6434e-03,  1.1965e-01,\n",
      "        -1.4251e-01, -7.7335e-02,  6.5381e-02,  2.2894e-01,  1.6913e-01,\n",
      "        -4.7191e-02,  1.5833e-01,  7.1202e-02,  4.4703e-02, -4.5435e-02,\n",
      "         9.3300e-02, -1.1701e-01,  4.1371e-01,  3.5706e-01,  1.3210e-01,\n",
      "         1.3935e-01, -1.0344e-01, -8.8519e-02, -1.4397e-01, -1.7125e-02,\n",
      "        -4.3861e-02,  9.5505e-02,  3.0059e-01, -3.0586e-01, -2.9731e-01,\n",
      "         4.5360e-03,  4.9629e-01,  4.9891e-02,  3.4414e-01,  1.5287e-02,\n",
      "         2.8124e-01, -2.2071e-01,  1.7462e+00,  1.2356e-02, -1.4539e-01,\n",
      "         7.8415e-02,  1.2502e-01,  1.6699e-02,  2.4823e-01, -1.8340e-01,\n",
      "         4.6021e-01,  3.2212e-01,  2.3975e-01, -4.0018e-02,  1.2354e-02,\n",
      "        -4.5039e-02,  1.2517e-01,  3.8175e-01,  1.9455e-01,  7.9882e-05,\n",
      "         3.2213e-01, -4.2299e-01, -2.5608e-01,  2.1356e-02, -2.3165e-01,\n",
      "        -1.2613e-02,  7.4803e-03,  1.6484e-01, -1.0356e-02, -1.9280e-01,\n",
      "         1.0410e-01,  8.7596e-02, -1.6314e-01, -7.2147e-02,  1.0347e-01,\n",
      "         1.9438e-01, -2.7588e-01,  5.5876e-01, -1.5294e-01, -9.0134e-03,\n",
      "         2.7256e-01, -6.7469e-02,  2.6406e-01,  1.7936e-01, -2.0995e-02,\n",
      "         2.8674e-01, -7.6322e-02,  4.6332e-01,  5.7285e-02,  1.6904e-01,\n",
      "        -4.7168e-02,  5.0974e-01,  2.4815e-01, -4.0165e-02, -7.0971e-03,\n",
      "         7.5506e-02, -1.0401e-01,  2.3993e-01, -2.0105e-01, -1.4279e-01,\n",
      "         3.8839e-01,  4.7292e-02, -1.4773e-01,  5.6040e-01,  1.8877e-01,\n",
      "        -2.9182e-02,  8.4681e-02,  1.2705e-02,  2.2226e-01, -3.3501e-02,\n",
      "        -1.2843e-01,  3.9450e-01,  1.1059e-01,  1.1330e-01,  2.2819e-02,\n",
      "         1.9717e-01, -4.1064e-02,  1.9359e-01,  2.5389e-01,  1.6845e-01,\n",
      "        -1.6516e-01,  4.2748e-01, -7.3921e-02, -6.5919e-02, -4.6100e-02,\n",
      "        -5.3108e-03, -1.3652e-01,  1.4109e-01,  5.2445e-02,  2.6868e-01,\n",
      "        -1.2549e-01,  1.1607e-01,  1.1015e-01, -1.2099e-01, -3.6625e-01,\n",
      "         2.4077e-01, -5.2464e-02, -1.3352e-02, -8.2583e-01,  1.8763e-01,\n",
      "         7.2441e-02, -1.2227e-02,  6.7365e-01,  6.5088e-02, -5.7020e-02,\n",
      "        -9.1223e-02, -7.6114e-02,  4.3872e-02,  1.9342e-01,  4.9467e-01,\n",
      "         7.1464e-02,  2.6926e-01,  1.6169e-01, -1.3850e-01, -5.5274e-01,\n",
      "        -2.1640e-01, -2.5027e-01,  1.7685e-01,  2.0592e-01, -1.3044e-01,\n",
      "         1.7712e-01,  2.1566e-01,  3.2794e-01,  1.8460e-01, -1.7768e-02,\n",
      "        -1.8338e-01, -1.8686e-01,  2.2537e-02, -1.4090e-01,  1.5759e-01,\n",
      "         7.3173e-02,  8.7164e-02, -8.3441e-03,  1.8688e-01,  2.6455e-01,\n",
      "         8.4719e-02,  1.7602e-01, -7.9884e-02,  7.7943e-02,  1.3058e-01,\n",
      "         2.4777e-02, -3.1255e-01, -1.1532e-01, -5.1967e-02,  2.4742e-01,\n",
      "         3.7210e-01,  5.4443e-01,  3.4146e-01,  6.4630e-01, -1.9522e-01,\n",
      "        -4.4525e-02,  2.3384e-01,  3.9411e-01,  3.5730e-02,  4.9153e-02,\n",
      "         3.0798e-01,  1.1016e-01,  1.8241e-01,  3.5172e-01, -2.6532e-01,\n",
      "         3.1758e-01,  8.1097e-02, -1.1333e-01,  8.8102e-02,  3.4142e-01,\n",
      "         4.0904e-02, -4.7199e-02,  4.0900e-01, -2.7151e-02,  8.6728e-03,\n",
      "         1.2937e-01,  2.1779e-02, -2.1941e-02,  1.2392e-01,  2.7016e-01,\n",
      "         2.0090e-01, -2.3312e-01,  1.4211e-02,  7.2290e-01, -5.8008e-02,\n",
      "        -2.6831e-01,  3.0274e-01, -3.6057e-03,  1.9413e-02,  4.4988e-01,\n",
      "         3.5734e-01, -6.6952e-02,  2.7208e-01, -2.6770e-03,  3.9239e-01,\n",
      "        -3.4453e-01, -3.1495e-03,  4.5359e-01,  9.6339e-02,  1.7189e-01,\n",
      "        -2.2400e-01, -7.5907e-02,  1.9020e-01,  2.3051e-01])), ('cls.predictions.decoder.weight', tensor([[ 0.0112,  0.1011,  0.0444,  ...,  0.0314, -0.0032, -0.1670],\n",
      "        [-0.0045, -0.0391,  0.0037,  ...,  0.0233, -0.0054, -0.0109],\n",
      "        [-0.0679, -0.0397, -0.0364,  ..., -0.0282, -0.0413, -0.0045],\n",
      "        ...,\n",
      "        [ 0.0034, -0.0647, -0.0567,  ..., -0.0489,  0.0013, -0.1270],\n",
      "        [ 0.0291,  0.0950,  0.0634,  ...,  0.0488, -0.0124, -0.1808],\n",
      "        [ 0.0464,  0.0704,  0.0567,  ...,  0.0335,  0.0045, -0.1737]])), ('cls.seq_relationship.weight', tensor([[ 1.4149e-02, -9.3360e-03, -1.5270e-02,  1.2861e-02, -9.4898e-02,\n",
      "          5.3826e-03,  2.1850e-02, -8.9140e-03, -5.1935e-03, -1.3856e-02,\n",
      "          8.5386e-03, -4.9050e-02, -1.3146e-02,  2.4047e-02,  1.8400e-02,\n",
      "          3.9365e-03, -8.6883e-05,  4.9051e-03,  1.5301e-02, -1.0465e-02,\n",
      "         -2.3854e-03, -1.8727e-03,  1.5637e-03, -4.6889e-03, -1.3793e-02,\n",
      "         -1.2758e-02,  2.8390e-02, -2.2156e-02,  2.1602e-02, -1.6264e-02,\n",
      "          6.5157e-04,  2.4862e-03,  6.4841e-04, -1.7045e-02, -1.9303e-02,\n",
      "          1.2863e-02, -2.2301e-02,  2.0869e-02, -1.0482e-02,  1.2586e-02,\n",
      "         -3.6887e-02, -3.1046e-02,  6.2273e-03,  1.6550e-02,  9.4041e-04,\n",
      "          2.6755e-02,  5.6826e-03, -3.3251e-02, -1.8972e-02,  9.4541e-03,\n",
      "          3.6295e-02,  1.6654e-02, -2.3709e-02, -6.0354e-04, -1.3749e-02,\n",
      "          6.1377e-03, -1.7358e-03, -5.2332e-03, -8.3774e-03, -3.0474e-02,\n",
      "         -2.5785e-02,  2.9242e-02, -1.6378e-02, -9.4389e-03, -1.1975e-02,\n",
      "         -2.8004e-02, -5.0414e-02,  1.0546e-03, -1.0394e-03, -9.7740e-02,\n",
      "         -2.9147e-02,  1.8321e-02,  7.0820e-02, -1.2797e-02, -1.5911e-03,\n",
      "         -2.5862e-03,  2.3302e-02,  1.1134e-02,  3.2771e-03,  1.3334e-04,\n",
      "          1.4866e-02,  1.4600e-02,  1.5661e-02, -1.0250e-02,  5.0527e-02,\n",
      "          4.2536e-02,  1.1118e-02, -1.2871e-02, -4.6751e-02, -2.6015e-02,\n",
      "          2.5800e-02, -2.0488e-02,  1.8081e-02, -1.4083e-02, -1.1828e-02,\n",
      "         -4.4065e-03, -1.8291e-02,  5.8286e-02,  6.5590e-02, -2.0476e-02,\n",
      "          2.9179e-02,  2.1222e-02,  2.0121e-02, -3.1126e-02,  2.3215e-02,\n",
      "         -1.9949e-02,  1.3618e-02,  9.6163e-03,  4.5886e-03,  3.6184e-02,\n",
      "         -1.2844e-02,  9.8641e-03,  1.2187e-02, -6.8923e-02,  1.1955e-02,\n",
      "          9.3615e-03,  1.9190e-02, -9.6378e-03, -5.1085e-02, -7.5836e-02,\n",
      "         -2.6977e-03,  2.0354e-02,  5.9022e-02, -5.3813e-03,  1.1054e-02,\n",
      "          4.2845e-03,  7.2964e-03,  1.2436e-02, -9.2556e-03, -7.3115e-03,\n",
      "         -7.3533e-03, -6.3398e-02,  2.7006e-02, -3.5044e-03,  9.8703e-03,\n",
      "         -5.4777e-02, -4.4330e-03,  1.4882e-02,  1.3916e-02,  4.4495e-02,\n",
      "         -5.7253e-03, -9.7403e-02,  5.1991e-02, -3.6822e-03,  1.0802e-03,\n",
      "          1.4873e-02,  1.5033e-02, -7.5260e-03, -7.5174e-03,  1.4679e-02,\n",
      "         -2.1614e-02,  2.9415e-02, -8.7418e-02,  4.4488e-02, -1.4136e-02,\n",
      "         -1.9645e-03,  6.6935e-02, -5.7244e-03, -2.9759e-02, -1.7434e-02,\n",
      "         -9.9135e-03, -8.4702e-03,  3.3589e-02, -1.4320e-02,  1.4543e-02,\n",
      "         -1.8720e-02, -1.5351e-02,  9.5206e-02,  3.9020e-03, -5.0672e-02,\n",
      "          8.3719e-03,  1.9958e-02,  1.9407e-02, -4.2013e-03, -2.2033e-02,\n",
      "         -5.9935e-02,  1.9026e-04, -9.2366e-03, -4.1507e-02,  2.6997e-02,\n",
      "         -1.7776e-02, -1.1904e-02, -1.1323e-02,  1.1694e-02,  5.1039e-02,\n",
      "         -7.2397e-03,  8.3267e-03, -7.6526e-02, -4.9549e-03,  4.4488e-02,\n",
      "         -5.1722e-02,  4.0651e-03, -1.0680e-02, -6.1278e-02, -5.7328e-04,\n",
      "         -1.9413e-02, -1.5113e-02, -3.0263e-03, -1.2702e-02,  1.2833e-02,\n",
      "         -2.3097e-02, -1.6534e-02, -1.6580e-02, -2.0108e-02,  1.2540e-02,\n",
      "         -6.9489e-03, -8.3155e-02, -6.8354e-03, -2.4222e-02,  4.5601e-03,\n",
      "         -2.4512e-02,  2.4299e-02, -7.2367e-04, -1.9846e-02, -1.2702e-02,\n",
      "         -1.4807e-03, -1.1170e-02,  6.6524e-03, -1.7646e-02, -2.8610e-03,\n",
      "         -3.3002e-02,  1.3088e-02,  1.6298e-02,  3.7982e-03, -1.8332e-02,\n",
      "          1.4741e-02,  4.7910e-02,  1.5125e-02,  2.3240e-02,  2.8199e-03,\n",
      "          9.0039e-03,  2.0067e-03, -1.7334e-02,  1.8306e-02,  4.8404e-03,\n",
      "         -6.8866e-03, -6.4056e-03,  9.9050e-03,  4.3610e-02, -1.0331e-02,\n",
      "         -1.1532e-03, -1.4687e-02,  1.7511e-03, -1.8042e-04, -3.1340e-03,\n",
      "          2.2651e-02,  9.1110e-03,  1.6652e-03, -4.3257e-03, -8.7736e-03,\n",
      "         -1.1193e-02,  1.6201e-02,  2.1381e-02, -1.4520e-02, -9.8739e-03,\n",
      "          4.9362e-02, -1.2769e-02,  4.2844e-02,  3.0521e-02,  1.3840e-03,\n",
      "          1.0403e-01, -8.3233e-03, -4.3859e-02,  1.0525e-02, -1.2337e-02,\n",
      "          1.9807e-02,  1.0792e-02,  8.0650e-03,  1.2119e-03, -6.1817e-03,\n",
      "         -4.9314e-02, -4.9849e-02, -6.8485e-03,  3.4316e-03,  8.5205e-03,\n",
      "         -2.9280e-02,  1.7438e-02, -3.1531e-02,  4.6348e-03,  2.1939e-03,\n",
      "         -1.0537e-02,  8.4493e-03, -3.0087e-02,  2.6562e-02, -1.7685e-03,\n",
      "         -5.1065e-02, -9.5206e-03, -1.8069e-03, -9.0617e-03,  1.3545e-02,\n",
      "         -1.4840e-02, -3.1261e-02, -5.1869e-02, -7.6375e-03,  7.6923e-03,\n",
      "          8.2730e-03,  4.1021e-02,  2.0067e-02, -8.9179e-03,  2.5239e-02,\n",
      "         -5.1106e-03,  1.5392e-02, -3.4629e-03, -5.6256e-02, -7.3775e-03,\n",
      "         -1.6470e-03,  1.8085e-03, -5.3341e-02,  7.2775e-03,  8.8445e-03,\n",
      "          2.4190e-02,  8.1740e-05, -7.3106e-02,  5.1620e-02, -2.2811e-02,\n",
      "         -5.4399e-02, -1.6009e-02, -1.0895e-02, -1.2429e-02, -1.5245e-02,\n",
      "          7.0942e-03,  2.0135e-02,  2.4077e-02, -1.0090e-02, -2.5032e-03,\n",
      "         -3.6420e-03, -9.5912e-03, -2.4194e-02, -6.1328e-02,  5.2180e-03,\n",
      "          1.8527e-02, -1.1580e-02,  9.3152e-03, -5.7296e-03, -9.5240e-03,\n",
      "          5.0034e-03, -2.2826e-03,  2.8955e-02, -1.1327e-02, -7.5920e-04,\n",
      "         -2.5853e-02, -1.3552e-02, -5.2007e-02, -5.6684e-03,  1.4383e-03,\n",
      "          3.5547e-03,  5.0647e-03,  3.7729e-02,  1.2570e-03, -1.1347e-02,\n",
      "          1.3655e-02,  2.9000e-02, -1.6558e-02,  6.0431e-03,  1.3832e-02,\n",
      "         -2.0558e-02, -2.3312e-03, -1.8054e-02, -1.0348e-02,  1.9638e-02,\n",
      "         -1.1876e-02, -4.7524e-03, -5.4949e-03,  1.0619e-02, -1.6490e-02,\n",
      "          1.4217e-03,  1.4069e-02, -1.7780e-02,  6.5183e-02,  6.0770e-02,\n",
      "          2.4879e-02,  6.8011e-03,  5.5573e-02, -2.4391e-02, -3.6535e-02,\n",
      "          1.2662e-02,  9.7612e-03,  6.4436e-03, -1.3501e-02, -7.0922e-02,\n",
      "         -1.0053e-02, -1.5967e-02,  6.7720e-03, -3.5599e-02],\n",
      "        [ 7.6937e-03, -7.5356e-03, -1.4586e-02, -1.5873e-02,  6.3163e-02,\n",
      "          3.7222e-02,  2.2261e-02, -8.8548e-03, -3.5633e-03, -1.3286e-02,\n",
      "          8.8509e-03,  3.3444e-02, -1.4159e-03,  2.2317e-02,  1.5350e-02,\n",
      "         -1.9702e-03, -7.8748e-04,  4.0287e-03,  1.4646e-02, -1.0675e-02,\n",
      "         -1.1346e-03,  5.2206e-03, -1.1804e-03, -4.5630e-03, -1.3487e-02,\n",
      "         -1.3485e-02, -4.4773e-03, -1.9885e-02, -4.7496e-04, -1.6216e-02,\n",
      "          1.6579e-03,  2.3671e-03,  6.7918e-04, -2.0922e-02, -2.0489e-02,\n",
      "          1.0445e-02, -4.1775e-02,  2.0243e-02, -9.6700e-03,  1.1843e-02,\n",
      "         -1.1710e-02, -2.9296e-02,  4.9668e-03,  1.5868e-02,  1.9198e-03,\n",
      "          2.6652e-02,  6.4245e-03,  1.6462e-02, -1.8827e-02,  1.1230e-02,\n",
      "         -7.1381e-02,  1.6009e-02,  2.9521e-02,  2.1296e-03, -1.1666e-02,\n",
      "          8.4026e-03, -1.7407e-03, -7.2183e-03, -1.0540e-02,  5.2799e-02,\n",
      "         -3.6967e-03, -7.6555e-02, -1.6421e-02, -2.0836e-02,  1.3960e-02,\n",
      "         -2.6864e-02,  6.3594e-02, -1.6565e-03,  1.7147e-04,  7.6262e-02,\n",
      "         -3.3795e-02,  1.6459e-02, -4.7510e-02, -3.0232e-03, -2.5392e-02,\n",
      "         -2.3178e-03, -2.5897e-02,  1.1521e-02,  2.9376e-03, -5.7135e-02,\n",
      "          1.5800e-02,  1.3967e-02,  9.4759e-03, -1.1981e-02, -5.3150e-02,\n",
      "         -9.6013e-02,  1.0951e-02, -1.3639e-02, -4.3864e-02, -2.7377e-02,\n",
      "          2.5831e-02,  1.9690e-02,  1.6673e-02, -2.0096e-02, -1.1497e-02,\n",
      "         -4.3544e-03,  1.3308e-02, -3.7966e-03,  1.7863e-02, -1.9798e-02,\n",
      "          2.1350e-02, -3.3463e-02,  1.9088e-02,  2.6376e-02,  1.3525e-02,\n",
      "         -2.1224e-02,  1.3528e-02,  1.0029e-02,  4.6868e-03,  3.5737e-02,\n",
      "         -1.2760e-02,  1.4258e-02,  6.4394e-03,  5.5516e-02,  1.1175e-02,\n",
      "          3.1208e-02, -2.1062e-02, -9.8604e-03,  1.0265e-01,  2.5645e-02,\n",
      "         -2.6703e-03,  2.3639e-02, -2.9428e-02, -3.0991e-03,  1.1499e-02,\n",
      "          5.2520e-03,  2.1661e-02,  1.5006e-02, -1.1628e-02, -1.1701e-03,\n",
      "         -9.0559e-03,  7.3043e-02,  2.5981e-02, -4.5556e-03,  1.3712e-02,\n",
      "          6.0604e-02, -2.7015e-03, -4.3489e-03,  1.3854e-02, -5.6333e-02,\n",
      "         -6.7231e-03,  8.3262e-02, -7.0148e-02, -2.7634e-03,  1.2367e-03,\n",
      "          1.4169e-02, -1.5812e-03, -8.1997e-03, -4.2336e-02,  1.5556e-02,\n",
      "         -1.8328e-02,  2.0578e-02,  5.1357e-02, -6.1730e-02, -1.5485e-02,\n",
      "         -1.8748e-03, -3.1274e-03, -4.3175e-03,  3.8148e-02, -1.4943e-02,\n",
      "         -9.3873e-03, -7.4083e-03,  2.8290e-02,  3.0726e-02,  1.4303e-02,\n",
      "         -1.7563e-02, -2.2918e-02, -6.9991e-02,  3.6649e-03, -1.2656e-02,\n",
      "          4.4449e-03,  1.9732e-02,  2.0476e-02, -3.5721e-03, -2.2006e-02,\n",
      "         -2.4291e-03, -2.1969e-02, -8.6254e-03, -1.4386e-02,  2.7269e-02,\n",
      "         -1.6823e-02, -8.8452e-03, -1.1924e-02,  9.5901e-03, -5.8047e-02,\n",
      "         -3.7280e-03,  7.5176e-03,  7.4044e-02, -4.1298e-03, -4.4693e-02,\n",
      "          4.5698e-02,  3.9948e-03, -1.2239e-02,  6.3611e-02,  6.7816e-03,\n",
      "         -1.9500e-02, -1.1591e-02, -8.6568e-03, -1.1155e-02,  1.1468e-02,\n",
      "         -2.1953e-02, -1.5839e-02, -1.8138e-02, -1.9098e-02,  1.1948e-02,\n",
      "         -7.0832e-03,  5.4616e-02, -5.9127e-03, -2.4921e-02, -6.5578e-04,\n",
      "          1.4345e-02, -1.0101e-03, -5.3845e-03, -1.7427e-02,  3.2266e-02,\n",
      "         -4.3344e-03,  2.8844e-02,  6.7750e-03,  1.2082e-02, -1.4446e-03,\n",
      "         -3.3370e-02,  1.4549e-02,  1.6074e-02,  3.5714e-03, -1.7781e-02,\n",
      "          6.6272e-03, -4.4961e-02,  1.4219e-02,  1.2610e-02,  2.0346e-03,\n",
      "          6.8592e-03,  2.7493e-03, -1.7411e-02,  2.1658e-02,  4.2769e-03,\n",
      "         -7.5285e-03, -6.9156e-03,  9.1583e-03, -3.0205e-02, -1.0561e-02,\n",
      "         -1.0875e-03, -1.3796e-02,  1.3924e-03,  3.2096e-02, -3.7165e-03,\n",
      "          2.2775e-02,  8.5082e-03,  4.1148e-04, -4.3357e-03, -8.7905e-03,\n",
      "         -1.0285e-02, -1.3214e-02,  2.0413e-02, -1.9806e-02, -8.2253e-03,\n",
      "         -4.9409e-02, -1.2483e-02, -1.7780e-02,  3.1472e-02,  7.7930e-03,\n",
      "         -5.9821e-02, -8.8293e-03,  2.2706e-02,  7.1578e-03, -1.1527e-02,\n",
      "          1.8560e-02,  1.0615e-02,  9.8814e-03,  9.4118e-04, -6.9985e-03,\n",
      "          3.9523e-02,  5.4361e-02, -5.0343e-03,  5.2431e-03,  2.6577e-02,\n",
      "         -1.6853e-03, -3.9316e-02, -3.1458e-02,  6.5274e-03,  6.2650e-03,\n",
      "         -1.2144e-02,  7.6226e-03, -3.0028e-02,  2.6008e-02, -2.1444e-03,\n",
      "          4.7070e-02, -9.9441e-03, -3.6042e-03, -1.1477e-02,  1.4883e-02,\n",
      "         -1.2320e-02,  6.6277e-03,  5.0806e-02, -4.5994e-02,  5.2667e-03,\n",
      "          1.0276e-02, -2.4138e-02,  1.6673e-02,  3.5878e-02,  2.2438e-02,\n",
      "         -5.0429e-03,  1.7387e-02, -4.5854e-03,  7.8076e-02, -7.0441e-03,\n",
      "         -2.2188e-03,  6.6869e-04,  1.3612e-02,  6.9618e-03,  9.8366e-03,\n",
      "          2.2955e-02,  1.9787e-02,  1.1408e-01, -6.3027e-02,  1.7903e-02,\n",
      "          7.9355e-02, -1.4776e-02, -9.1962e-03, -1.2996e-02, -9.6100e-03,\n",
      "          7.6954e-03,  1.9687e-02, -5.7211e-02, -4.5164e-03, -2.8594e-03,\n",
      "         -5.2185e-03, -1.1032e-02,  1.1968e-02,  5.9730e-02,  7.4316e-03,\n",
      "          6.6947e-03, -1.1915e-02,  7.5449e-03, -3.9171e-03, -9.2350e-03,\n",
      "          4.5402e-03,  1.8284e-03, -2.7651e-02, -1.0908e-02, -5.0321e-03,\n",
      "          1.2449e-02, -1.0647e-02,  8.1451e-02, -3.4300e-03, -1.5252e-03,\n",
      "          7.4259e-03, -4.3320e-02,  2.1415e-03,  1.1375e-03,  2.5006e-04,\n",
      "          1.2269e-02,  1.9478e-02, -1.3031e-02,  8.2410e-03,  1.6055e-02,\n",
      "         -3.7910e-02, -4.1236e-02, -1.7879e-02, -1.0532e-02, -4.5681e-02,\n",
      "         -8.1626e-03, -5.3781e-03,  1.3051e-02,  1.5258e-02,  2.5654e-02,\n",
      "         -8.0370e-04, -1.8817e-02, -1.7944e-02, -8.8009e-02, -1.8760e-02,\n",
      "         -4.1001e-02,  7.0123e-03, -7.9887e-02, -2.5118e-02,  1.9732e-02,\n",
      "          1.8932e-02,  1.2514e-02,  2.5233e-03, -2.2262e-02,  7.2499e-02,\n",
      "         -6.5039e-03, -1.6109e-02,  6.1614e-03,  1.0876e-02]])), ('cls.seq_relationship.bias', tensor([ 0.0092, -0.0092]))])\n",
      "../03_transformer_tutorial_1st_part/BERT_tutorial_and_attention_visualization/bert_state_dict/bert.model.epoch.3 loaded for training!\n"
     ]
    }
   ],
   "source": [
    "# 加载模型\n",
    "model = bert_model\n",
    "dir_path=config[\"output_path\"]\n",
    "checkpoint_dir = find_most_recent_state_dict(dir_path)\n",
    "checkpoint = torch.load(checkpoint_dir)\n",
    "print(\"checkpoint[\\\"model_state_dict\\\"]\", checkpoint[\"model_state_dict\"])\n",
    "model.load_state_dict(checkpoint[\"model_state_dict\"], strict=False)\n",
    "torch.cuda.empty_cache()\n",
    "model.to(device)\n",
    "print(\"{} loaded for training!\".format(checkpoint_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_epoch = 0\n",
    "train_epoches = 1\n",
    "all_loss = []\n",
    "threshold = 0\n",
    "patient = 10\n",
    "best_f1 = 0\n",
    "dynamic_lr = config[\"lr\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_state_dict(model, epoch, dir_path=\"./output\", file_path=\"bert.model\"):\n",
    "    if not os.path.exists(dir_path):\n",
    "        os.mkdir(dir_path)\n",
    "    save_path = dir_path+ \"/\" + file_path + \".epoch.{}\".format(str(epoch))\n",
    "    model.to(\"cpu\")\n",
    "    torch.save({\"model_state_dict\": model.state_dict()}, save_path)\n",
    "    print(\"{} saved!\".format(save_path))\n",
    "    model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_loss(predictions, labels, num_class=2, ignore_index=None):\n",
    "    if ignore_index is None:\n",
    "        loss_func = CrossEntropyLoss()\n",
    "    else:\n",
    "        loss_func = CrossEntropyLoss(ignore_index=ignore_index)\n",
    "    return loss_func(predictions.view(-1, num_class), labels.view(-1))\n",
    "\n",
    "def get_mlm_accuracy(predictions, labels):\n",
    "    predictions = torch.argmax(predictions, dim=-1, keepdim=False)\n",
    "    mask = (labels > 0).to(device)\n",
    "    mlm_accuracy = torch.sum((predictions == labels) * mask).float()\n",
    "    mlm_accuracy /= (torch.sum(mask).float() + 1e-8)\n",
    "    return mlm_accuracy.item()\n",
    "\n",
    "def padding(output_dic_lis):\n",
    "    bert_input = [i[\"bert_input\"] for i in output_dic_lis]\n",
    "    bert_label = [i[\"bert_label\"] for i in output_dic_lis]\n",
    "    segment_label = [i[\"segment_label\"] for i in output_dic_lis]\n",
    "    bert_input = torch.nn.utils.rnn.pad_sequence(bert_input, batch_first=True)\n",
    "    bert_label = torch.nn.utils.rnn.pad_sequence(bert_label, batch_first=True)\n",
    "    segment_label = torch.nn.utils.rnn.pad_sequence(segment_label, batch_first=True)\n",
    "    is_next = torch.cat([i[\"is_next\"] for i in output_dic_lis])\n",
    "    return {\"bert_input\": bert_input,\n",
    "            \"bert_label\": bert_label,\n",
    "            \"segment_label\": segment_label,\n",
    "            \"is_next\": is_next}\n",
    "\n",
    "def iteration(epoch, data_loader, train=True, df_path=\"./my_output_path/df_log.pickle\"):\n",
    "    if not os.path.isfile(df_path) and epoch != 0:\n",
    "        raise RuntimeError(\"log DataFrame path not found and can't create a new one because we're not training from scratch!\")\n",
    "    if not os.path.isfile(df_path) and epoch == 0:\n",
    "        df = pd.DataFrame(columns=[\"epoch\", \"train_next_sen_loss\", \"train_mlm_loss\",\n",
    "                                    \"train_next_sen_acc\", \"train_mlm_acc\",\n",
    "                                    \"test_next_sen_loss\", \"test_mlm_loss\",\n",
    "                                    \"test_next_sen_acc\", \"test_mlm_acc\"\n",
    "                                    ])\n",
    "        df.to_pickle(df_path)\n",
    "        print(\"log DataFrame created!\")\n",
    "\n",
    "    str_code = \"train\" if train else \"test\"\n",
    "\n",
    "    # Setting the tqdm progress bar\n",
    "    data_iter = tqdm.tqdm(enumerate(data_loader),\n",
    "                            desc=\"EP_%s:%d\" % (str_code, epoch),\n",
    "                            total=len(data_loader),\n",
    "                            bar_format=\"{l_bar}{r_bar}\")\n",
    "\n",
    "    total_next_sen_loss = 0\n",
    "    total_mlm_loss = 0\n",
    "    total_next_sen_acc = 0\n",
    "    total_mlm_acc = 0\n",
    "    total_element = 0\n",
    "\n",
    "    for i, data in data_iter:\n",
    "        # print('IDX of data_iter:', i)\n",
    "        data = padding(data)\n",
    "        # 0. batch_data will be sent into the device(GPU or cpu)\n",
    "        data = {key: value.to(device) for key, value in data.items()}\n",
    "        positional_enc = self_positional_enc[:, :data[\"bert_input\"].size()[-1], :].to(device)\n",
    "\n",
    "        # 1. forward the next_sentence_prediction and masked_lm model\n",
    "        mlm_preds, next_sen_preds = bert_model.forward(input_ids=data[\"bert_input\"],\n",
    "                                                            positional_enc=positional_enc,\n",
    "                                                            token_type_ids=data[\"segment_label\"])\n",
    "        #print(\"mlm_preds\", mlm_preds.shape, mlm_preds[0][0])\n",
    "        mlm_acc = get_mlm_accuracy(mlm_preds, data[\"bert_label\"])\n",
    "        next_sen_acc = next_sen_preds.argmax(dim=-1, keepdim=False).eq(data[\"is_next\"]).sum().item()\n",
    "        mlm_loss = compute_loss(mlm_preds, data[\"bert_label\"], vocab_size, ignore_index=0)\n",
    "        next_sen_loss = compute_loss(next_sen_preds, data[\"is_next\"])\n",
    "        loss = mlm_loss + next_sen_loss\n",
    "\n",
    "\n",
    "        # 3. backward and optimization only in train\n",
    "        if train:\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            # for param in model.parameters():\n",
    "            #     print(param.grad.data.sum())\n",
    "            optimizer.step()\n",
    "\n",
    "\n",
    "        total_next_sen_loss += next_sen_loss.item()\n",
    "        total_mlm_loss += mlm_loss.item()\n",
    "        total_next_sen_acc += next_sen_acc\n",
    "        total_element += data[\"is_next\"].nelement()\n",
    "        total_mlm_acc += mlm_acc\n",
    "\n",
    "        if train:\n",
    "            log_dic = {\n",
    "                \"epoch\": epoch,\n",
    "                \"train_next_sen_loss\": total_next_sen_loss / (i + 1),\n",
    "                \"train_mlm_loss\": total_mlm_loss / (i + 1),\n",
    "                \"train_next_sen_acc\": total_next_sen_acc / total_element,\n",
    "                \"train_mlm_acc\": total_mlm_acc / (i + 1),\n",
    "                \"test_next_sen_loss\": 0, \"test_mlm_loss\": 0,\n",
    "                \"test_next_sen_acc\": 0, \"test_mlm_acc\": 0\n",
    "            }\n",
    "\n",
    "        else:\n",
    "            log_dic = {\n",
    "                \"epoch\": epoch,\n",
    "                \"test_next_sen_loss\": total_next_sen_loss / (i + 1),\n",
    "                \"test_mlm_loss\": total_mlm_loss / (i + 1),\n",
    "                \"test_next_sen_acc\": total_next_sen_acc / total_element,\n",
    "                \"test_mlm_acc\": total_mlm_acc / (i + 1),\n",
    "                \"train_next_sen_loss\": 0, \"train_mlm_loss\": 0,\n",
    "                \"train_next_sen_acc\": 0, \"train_mlm_acc\": 0\n",
    "            }\n",
    "\n",
    "\n",
    "        if i % 10 == 0:\n",
    "            data_iter.write(str({k: v for k, v in log_dic.items() if v != 0 and k != \"epoch\"}))\n",
    "\n",
    "    if train:\n",
    "        df = pd.read_pickle(df_path)\n",
    "        df = df._append([log_dic])\n",
    "        #df = df.concat([log_dic], ignore_index=True)\n",
    "        df.reset_index(inplace=True, drop=True)\n",
    "        df.to_pickle(df_path)\n",
    "    else:\n",
    "        log_dic = {k: v for k, v in log_dic.items() if v != 0 and k != \"epoch\"}\n",
    "        df = pd.read_pickle(df_path)\n",
    "        df.reset_index(inplace=True, drop=True)\n",
    "        for k, v in log_dic.items():\n",
    "            df.at[epoch, k] = v\n",
    "        df.to_pickle(df_path)\n",
    "        return float(log_dic[\"test_next_sen_loss\"])+float(log_dic[\"test_mlm_loss\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train start with learning rate 2e-06\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_train:1:   0%|| 0/64 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_train:1:   2%|| 1/64 [00:02<02:32,  2.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train_next_sen_loss': 0.6734557747840881, 'train_mlm_loss': 9.805048942565918, 'train_next_sen_acc': 1.0, 'train_mlm_acc': 0.03846153989434242}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_train:1:  17%|| 11/64 [00:21<01:41,  1.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train_next_sen_loss': 0.6956638531251387, 'train_mlm_loss': 9.84447158466686, 'train_next_sen_acc': 0.5454545454545454, 'train_mlm_acc': 0.030750407921996983}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_train:1:  33%|| 21/64 [00:40<01:36,  2.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train_next_sen_loss': 0.693936410404387, 'train_mlm_loss': 9.829984210786366, 'train_next_sen_acc': 0.5238095238095238, 'train_mlm_acc': 0.030187233883355345}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_train:1:  48%|| 31/64 [01:08<01:24,  2.57s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train_next_sen_loss': 0.6959477086221019, 'train_mlm_loss': 9.844615013368669, 'train_next_sen_acc': 0.5161290322580645, 'train_mlm_acc': 0.02787123423730654}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_train:1:  64%|| 41/64 [01:37<01:18,  3.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train_next_sen_loss': 0.6946371386690837, 'train_mlm_loss': 9.850404902202326, 'train_next_sen_acc': 0.5609756097560976, 'train_mlm_acc': 0.028517540237616476}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_train:1:  80%|| 51/64 [02:17<00:51,  3.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train_next_sen_loss': 0.6920523503247429, 'train_mlm_loss': 9.843021804211187, 'train_next_sen_acc': 0.5686274509803921, 'train_mlm_acc': 0.03230088988465129}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_train:1:  95%|| 61/64 [02:46<00:07,  2.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train_next_sen_loss': 0.6922596634411421, 'train_mlm_loss': 9.840110685004563, 'train_next_sen_acc': 0.5573770491803278, 'train_mlm_acc': 0.032857678349694754}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_train:1: 100%|| 64/64 [02:55<00:00,  2.74s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train end: 1\n",
      "./my_output_path/bert.model.epoch.1 saved!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_test:1:   2%|| 1/59 [00:00<00:46,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'test_next_sen_loss': 0.5642930865287781, 'test_mlm_loss': 9.849289894104004, 'test_next_sen_acc': 1.0, 'test_mlm_acc': 0.03741496428847313}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_test:1:  19%|| 11/59 [00:08<00:36,  1.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'test_next_sen_loss': 0.6396935690533031, 'test_mlm_loss': 9.808637445623225, 'test_next_sen_acc': 0.7272727272727273, 'test_mlm_acc': 0.038757564330642875}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_test:1:  36%|| 21/59 [00:16<00:22,  1.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'test_next_sen_loss': 0.6826409583999997, 'test_mlm_loss': 9.821704546610514, 'test_next_sen_acc': 0.5714285714285714, 'test_mlm_acc': 0.0352994975234781}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_test:1:  53%|| 31/59 [00:21<00:10,  2.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'test_next_sen_loss': 0.6717215930261919, 'test_mlm_loss': 9.810143193890971, 'test_next_sen_acc': 0.6129032258064516, 'test_mlm_acc': 0.035624984711889296}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_test:1:  69%|| 41/59 [00:28<00:12,  1.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'test_next_sen_loss': 0.6723795605868828, 'test_mlm_loss': 9.81268936250268, 'test_next_sen_acc': 0.6097560975609756, 'test_mlm_acc': 0.03434747392765996}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_test:1:  86%|| 51/59 [00:34<00:04,  1.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'test_next_sen_loss': 0.6891432965503019, 'test_mlm_loss': 9.819493144166236, 'test_next_sen_acc': 0.5490196078431373, 'test_mlm_acc': 0.03209359924692441}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "EP_test:1: 100%|| 59/59 [00:39<00:00,  1.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eval_result: 1 nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "start_epoch = 1\n",
    "train_epoches = 1\n",
    "for epoch in range(start_epoch, start_epoch + train_epoches):\n",
    "    print(\"train start with learning rate {}\".format(str(dynamic_lr)))\n",
    "\n",
    "    bert_model.train()\n",
    "    iteration(epoch, train_dataloader, train=True)\n",
    "\n",
    "    print(\"train end:\", epoch)\n",
    "\n",
    "    save_state_dict(bert_model, epoch, dir_path=config[\"my_output_path\"], file_path=\"bert.model\")\n",
    "    \n",
    "    eval_result = 0;\n",
    "    bert_model.eval()\n",
    "    with torch.no_grad():\n",
    "        eval_result= iteration(epoch, test_dataloader, train=False)\n",
    "    print(\"eval_result:\", epoch, eval_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForPreTraining(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(32162, 384, padding_idx=0)\n",
       "      (token_type_embeddings): Embedding(256, 384)\n",
       "      (LayerNorm): BertLayerNorm()\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-5): 6 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=384, out_features=384, bias=True)\n",
       "              (key): Linear(in_features=384, out_features=384, bias=True)\n",
       "              (value): Linear(in_features=384, out_features=384, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=384, out_features=384, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=384, out_features=1536, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=384, out_features=384, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (cls): BertPreTrainingHeads(\n",
       "    (predictions): BertLMPredictionHead(\n",
       "      (transform): BertPredictionHeadTransform(\n",
       "        (dense): Linear(in_features=384, out_features=384, bias=True)\n",
       "        (LayerNorm): BertLayerNorm()\n",
       "      )\n",
       "      (decoder): Linear(in_features=384, out_features=32162, bias=False)\n",
       "    )\n",
       "    (seq_relationship): Linear(in_features=384, out_features=2, bias=True)\n",
       "  )\n",
       "  (next_loss_func): CrossEntropyLoss()\n",
       "  (mlm_loss_func): CrossEntropyLoss()\n",
       ")"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_model.eval()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
